https://github.com/python/cpython/commit/7bfa97f4dca58ab45def2ca0a1e8088997a4fa0b
commit: 7bfa97f4dca58ab45def2ca0a1e8088997a4fa0b
branch: main
author: Pablo Galindo Salgado <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-09-12T14:30:00+01:00
summary:
gh-153569: hide tokenizer state behind token and source operations (#156654)
Make pegen and _tokenize consume tokenizer operations instead of accessing the
tokenizer layout. Return token kind, span, location, and raw-string context
together, so parser actions use the context of the cached token.
Represent persistent scanner positions as offsets and remove pointer
save/restore bookkeeping when input storage grows.
files:
A Parser/tokenizer/api.c
A Parser/tokenizer/types.h
D Parser/lexer/buffer.c
D Parser/lexer/buffer.h
M Lib/test/test_fstring.py
M Lib/test/test_tstring.py
M Makefile.pre.in
M PCbuild/_freeze_module.vcxproj
M PCbuild/_freeze_module.vcxproj.filters
M PCbuild/pythoncore.vcxproj
M PCbuild/pythoncore.vcxproj.filters
M Parser/action_helpers.c
M Parser/lexer/lexer.c
M Parser/lexer/lexer.h
M Parser/lexer/lexer_internal.h
M Parser/lexer/number.c
M Parser/lexer/state.c
M Parser/lexer/state.h
M Parser/lexer/string.c
M Parser/pegen.c
M Parser/pegen.h
M Parser/pegen_errors.c
M Parser/string_parser.c
M Parser/tokenizer/decoder.c
M Parser/tokenizer/helpers.c
M Parser/tokenizer/helpers.h
M Parser/tokenizer/reader.c
M Parser/tokenizer/source.h
M Parser/tokenizer/tokenizer.h
M Python/Python-tokenize.c
M Tools/peg_generator/pegen/build.py
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 460adde4956a9f2..201ae6e24794933 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -1891,6 +1891,9 @@ def __format__(self, format):
self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
+ self.assertEqual(rf"{UnchangedFormat():{f'\xFF'}}\n", 'ÿ\\n')
+ self.assertEqual(f"{UnchangedFormat():{rf'\xFF'}}\n", '\\xFF\n')
+
# Test continuation character in format specs
self.assertEqual(f"""{UnchangedFormat():{'a'\
'b'}}""", 'ab')
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 75d7085c6731d95..854860b5ea43065 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -159,6 +159,14 @@ def test_raw_tstrings(self):
t = tr"{path}\Documents"
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
+ value = 42
+ t = rt"{value:{f'\xFF'}}\n"
+ self.assertTStringEqual(
+ t, ("", "\\n"), [(value, "value", None, 'ÿ')])
+ t = t"{value:{rf'\xFF'}}\n"
+ self.assertTStringEqual(
+ t, ("", "\n"), [(value, "value", None, '\\xFF')])
+
def test_template_concatenation(self):
# Test template + template
t1 = t"Hello, "
@@ -226,6 +234,10 @@ def test_syntax_errors(self):
("t'{x=!}'", "t-string: missing conversion character"),
("t'{x!z}'", "t-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
+ ("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
+ "expected 's', 'r', or 'a'"),
+ ("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
+ "expected 's', 'r', or 'a'"),
("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
"without parentheses"),
("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 78a486623181fa8..166087f32dff187 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -394,13 +394,13 @@ PEGEN_OBJS= \
Parser/peg_api.o
TOKENIZER_OBJS= \
- Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
Parser/tokenizer/cursor.o \
Parser/tokenizer/decoder.o \
+ Parser/tokenizer/api.o \
Parser/tokenizer/reader.o \
Parser/tokenizer/source.o \
Parser/tokenizer/helpers.o
@@ -411,7 +411,6 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h
TOKENIZER_HEADERS= \
- Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
@@ -419,6 +418,7 @@ TOKENIZER_HEADERS= \
Parser/tokenizer/reader.h \
Parser/tokenizer/reader_internal.h \
Parser/tokenizer/source.h \
+ Parser/tokenizer/types.h \
Parser/tokenizer/tokenizer.h \
Parser/tokenizer/helpers.h
@@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h
$(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h
$(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c
$(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h
$(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h
$(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h
$(srcdir)/Modules/_testlimitedcapi/parts.h
$(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
$(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h
$(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h
$(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
$(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h
$(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h
$(srcdir)/Modules/_testinternalcapi/test_targets.h
$(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h
$(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h
$(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h
$(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h
$(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 469fd77cc8be9dc..70c54e0e41efc63 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -181,12 +181,12 @@
<ClCompile Include="..\Parser\action_helpers.c" />
<ClCompile Include="..\Parser\string_parser.c" />
<ClCompile Include="..\Parser\token.c" />
- <ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
+ <ClCompile Include="..\Parser\tokenizer\api.c" />
<ClCompile Include="..\Parser\tokenizer\reader.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
diff --git a/PCbuild/_freeze_module.vcxproj.filters
b/PCbuild/_freeze_module.vcxproj.filters
index 976c99b7d24bdfd..b0799b8dc9ecddb 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -469,15 +469,15 @@
<ClCompile Include="..\Parser\lexer\string.c">
<Filter>Source Files</Filter>
</ClCompile>
- <ClCompile Include="..\Parser\lexer\buffer.c">
- <Filter>Source Files</Filter>
- </ClCompile>
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\decoder.c">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\Parser\tokenizer\api.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="..\Parser\tokenizer\reader.c">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 79dfc9ccf39ec26..cd37db739301923 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -423,10 +423,10 @@
<ClInclude Include="..\Parser\lexer\state.h" />
<ClInclude Include="..\Parser\lexer\lexer.h" />
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
- <ClInclude Include="..\Parser\lexer\buffer.h" />
<ClInclude Include="..\Parser\tokenizer\cursor.h" />
<ClInclude Include="..\Parser\tokenizer\reader.h" />
<ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
+ <ClInclude Include="..\Parser\tokenizer\types.h" />
<ClInclude Include="..\Parser\tokenizer\source.h" />
<ClInclude Include="..\Parser\tokenizer\helpers.h" />
<ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -593,10 +593,10 @@
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
- <ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
+ <ClCompile Include="..\Parser\tokenizer\api.c" />
<ClCompile Include="..\Parser\tokenizer\reader.c" />
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
<ClCompile Include="..\Parser\token.c" />
diff --git a/PCbuild/pythoncore.vcxproj.filters
b/PCbuild/pythoncore.vcxproj.filters
index 765b4d46b12dd00..e4b9039eec13fc0 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -330,9 +330,6 @@
<ClInclude Include="..\Parser\lexer\lexer_internal.h">
<Filter>Parser</Filter>
</ClInclude>
- <ClInclude Include="..\Parser\lexer\buffer.h">
- <Filter>Parser</Filter>
- </ClInclude>
<ClInclude Include="..\Parser\tokenizer\cursor.h">
<Filter>Parser</Filter>
</ClInclude>
@@ -342,6 +339,9 @@
<ClInclude Include="..\Parser\tokenizer\reader_internal.h">
<Filter>Parser</Filter>
</ClInclude>
+ <ClInclude Include="..\Parser\tokenizer\types.h">
+ <Filter>Parser</Filter>
+ </ClInclude>
<ClInclude Include="..\Parser\tokenizer\source.h">
<Filter>Parser</Filter>
</ClInclude>
@@ -1361,9 +1361,6 @@
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Parser</Filter>
</ClCompile>
- <ClCompile Include="..\Parser\lexer\buffer.c">
- <Filter>Parser</Filter>
- </ClCompile>
<ClCompile Include="..\Parser\tokenizer\cursor.c">
<Filter>Parser</Filter>
</ClCompile>
@@ -1373,6 +1370,9 @@
<ClCompile Include="..\Parser\tokenizer\decoder.c">
<Filter>Parser</Filter>
</ClCompile>
+ <ClCompile Include="..\Parser\tokenizer\api.c">
+ <Filter>Parser</Filter>
+ </ClCompile>
<ClCompile Include="..\Parser\tokenizer\reader.c">
<Filter>Parser</Filter>
</ClCompile>
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
index 8ca6898711f72b0..3e4d463b36ab872 100644
--- a/Parser/action_helpers.c
+++ b/Parser/action_helpers.c
@@ -1004,8 +1004,20 @@ result_token_with_metadata(Parser *p, void *result,
PyObject *metadata)
static char
formatted_string_prefix(const Parser *p)
{
- const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
- return state == NULL ? 'f' : _PyLexer_StringPrefix(state->kind);
+ int nested = 0;
+ for (int i = p->mark - 1; i >= 0; i--) {
+ int type = p->tokens[i]->type;
+ if (type == FSTRING_END || type == TSTRING_END) {
+ nested++;
+ }
+ else if (type == FSTRING_START || type == TSTRING_START) {
+ if (nested == 0) {
+ return type == TSTRING_START ? 't' : 'f';
+ }
+ nested--;
+ }
+ }
+ Py_UNREACHABLE();
}
ResultTokenWithMetadata *
@@ -1352,7 +1364,7 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw,
expr_ty constant, Token* tok
static asdl_expr_seq *
_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
- Token *b, ftstring_kind string_kind)
+ Token *b, int is_tstring)
{
Py_ssize_t n_items = asdl_seq_LEN(raw_expressions);
Py_ssize_t total_items = n_items;
@@ -1384,7 +1396,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq
*raw_expressions,
asdl_expr_seq *values = item->v.JoinedStr.values;
if (asdl_seq_LEN(values) != 2) {
PyErr_Format(PyExc_SystemError,
- _PyLexer_IsTString(string_kind)
+ is_tstring
? "unexpected TemplateStr node without debug data
in t-string at line %d"
: "unexpected JoinedStr node without debug data
in f-string at line %d",
item->lineno);
@@ -1396,7 +1408,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq
*raw_expressions,
asdl_seq_SET(seq, index++, first);
expr_ty second = asdl_seq_GET(values, 1);
- assert((_PyLexer_IsTString(string_kind) &&
+ assert((is_tstring &&
second->kind == Interpolation_kind) ||
second->kind == FormattedValue_kind);
asdl_seq_SET(seq, index++, second);
@@ -1440,7 +1452,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq
*raw_expressions,
expr_ty
_PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
Token *b) {
- asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions,
b, TSTRING);
+ asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions,
b, 1);
if (resized_exprs == NULL) {
return NULL;
}
@@ -1452,7 +1464,7 @@ _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq
*raw_expressions, Token
expr_ty
_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions,
Token*b) {
- asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions,
b, FSTRING);
+ asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions,
b, 0);
if (resized_exprs == NULL) {
return NULL;
}
@@ -1468,8 +1480,7 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p,
Token* tok) {
return NULL;
}
- const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
- int is_raw = state != NULL && _PyLexer_IsRawString(state->kind);
+ int is_raw = tok->is_raw;
PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
if (str == NULL) {
@@ -2051,13 +2062,14 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq
*dots, expr_ty module)
return -1;
}
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
int res = _PyErr_EmitSyntaxWarning(msg,
- p->tok->filename,
+ info.filename,
module->lineno,
module->col_offset + 1,
module->end_lineno,
module->end_col_offset + 1,
- p->tok->module);
+ info.module);
Py_DECREF(msg);
return res;
}
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
deleted file mode 100644
index ead973b0e85234b..000000000000000
--- a/Parser/lexer/buffer.c
+++ /dev/null
@@ -1,29 +0,0 @@
-#include "Python.h"
-#include "buffer.h"
-#include "state.h"
-
-void
-_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
- _PyLexer_BufferPointers *pointers)
-{
- pointers->buf_from_base = tok->buf - base;
- pointers->cur_from_buf = tok->cur - tok->buf;
- pointers->inp_from_buf = tok->inp - tok->buf;
- pointers->start_from_buf = tok->start == NULL
- ? -1 : tok->start - tok->buf;
- pointers->line_start_from_buf = tok->line_start == NULL
- ? -1 : tok->line_start - tok->buf;
-}
-
-void
-_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
- const _PyLexer_BufferPointers *pointers)
-{
- tok->buf = base + pointers->buf_from_base;
- tok->cur = tok->buf + pointers->cur_from_buf;
- tok->inp = tok->buf + pointers->inp_from_buf;
- tok->start = pointers->start_from_buf < 0
- ? NULL : tok->buf + pointers->start_from_buf;
- tok->line_start = pointers->line_start_from_buf < 0
- ? NULL : tok->buf + pointers->line_start_from_buf;
-}
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
deleted file mode 100644
index af57732a69363f0..000000000000000
--- a/Parser/lexer/buffer.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef _LEXER_BUFFER_H_
-#define _LEXER_BUFFER_H_
-
-#include "pyport.h"
-
-struct tok_state;
-
-typedef struct {
- Py_ssize_t buf_from_base;
- Py_ssize_t cur_from_buf;
- Py_ssize_t inp_from_buf;
- Py_ssize_t start_from_buf;
- Py_ssize_t line_start_from_buf;
-} _PyLexer_BufferPointers;
-
-void _PyLexer_SaveBufferPointers(
- struct tok_state *, const char *, _PyLexer_BufferPointers *);
-void _PyLexer_RestoreBufferPointers(
- struct tok_state *, char *, const _PyLexer_BufferPointers *);
-
-#endif
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index c5157c1c021e012..110a225750f0550 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -23,43 +23,33 @@ contains_null_bytes(const char* str, size_t size)
return memchr(str, 0, size) != NULL;
}
-/* Get next char, updating state; error code goes into tok->done */
int
-_PyLexer_nextc(struct tok_state *tok)
+_PyLexer_refill(struct tok_state *tok)
{
- int rc;
- for (;;) {
- if (tok->cur != tok->inp) {
- if (tok->cur - tok->line_start >= INT_MAX) {
- tok->done = E_COLUMNOVERFLOW;
- return EOF;
- }
- return Py_CHARMASK(*tok->cur++); /* Fast path */
- }
- if (tok->done != E_OK) {
- return EOF;
- }
- rc = _PyTok_ReaderUnderflow(tok);
+ if (tok->done != E_OK) {
+ return 0;
+ }
+ int rc = _PyTok_ReaderUnderflow(tok);
#if defined(Py_DEBUG)
- if (tok->debug) {
- fprintf(stderr, "line[%d] = ", tok->lineno);
- _PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
- fprintf(stderr, " tok->done = %d\n", tok->done);
- }
+ if (tok->debug) {
+ fprintf(stderr, "line[%d] = ", tok->lineno);
+ _PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok,
tok->cur),
+ tok->inp - tok->cur);
+ fprintf(stderr, " tok->done = %d\n", tok->done);
+ }
#endif
- if (!rc) {
- tok->cur = tok->inp;
- return EOF;
- }
- tok->line_start = tok->cur;
-
- if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
- _PyTokenizer_syntaxerror(tok, "source code cannot contain null
bytes");
- tok->cur = tok->inp;
- return EOF;
- }
+ if (!rc) {
+ tok->cur = tok->inp;
+ return 0;
}
- Py_UNREACHABLE();
+ tok->line_start = tok->cur;
+ if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start),
+ tok->inp - tok->line_start)) {
+ _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
+ tok->cur = tok->inp;
+ return 0;
+ }
+ return 1;
}
/* Back-up one character */
@@ -67,10 +57,10 @@ void
_PyLexer_backup(struct tok_state *tok, int c)
{
if (c != EOF) {
- if (--tok->cur < tok->buf) {
+ if (--tok->cur < tok->buf_offset) {
Py_FatalError("tokenizer beginning of buffer");
}
- if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
+ if ((int)(unsigned char)*_PyLexer_BufferPointer(tok, tok->cur) !=
Py_CHARMASK(c)) {
Py_FatalError("tok_backup: wrong character");
}
}
@@ -88,7 +78,7 @@ verify_identifier(struct tok_state *tok)
PyObject *s;
if (tok_failed(tok))
return 0;
- s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
+ s = PyUnicode_DecodeUTF8(_PyLexer_BufferPointer(tok, tok->start), tok->cur
- tok->start, NULL);
if (s == NULL) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
tok->done = E_DECODE;
@@ -113,7 +103,7 @@ verify_identifier(struct tok_state *tok)
tok->done = E_ERROR;
return 0;
}
- tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
+ tok->cur = tok->start + PyBytes_GET_SIZE(s);
}
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
@@ -162,10 +152,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
int c;
int blankline, nonascii;
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
nextline:
- tok->start = NULL;
+ tok->start = -1;
tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
blankline = 0;
@@ -271,7 +261,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
tok->start = tok->cur;
tok->start_loc = (_PyTok_Loc){
- tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1};
+ tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1};
/* Return pending indents/dedents */
if (tok->pendin != 0) {
@@ -285,7 +275,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
}
else {
if (tok->tok_extra_tokens) {
- p_start = tok->buf;
+ p_start = tok->buf_offset;
p_end = tok->cur;
}
tok->pendin--;
@@ -298,16 +288,16 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
tok_backup(tok, c);
again:
- tok->start = NULL;
+ tok->start = -1;
/* Skip spaces */
do {
c = tok_nextc(tok);
} while (c == ' ' || c == '\t' || c == '\014');
/* Set start of current token */
- tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
+ tok->start = tok->cur - 1;
tok->start_loc = (_PyTok_Loc){
- tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1};
+ tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) - 1 : -1};
/* Skip comment, unless it's a type comment */
if (c == '#') {
@@ -321,7 +311,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
}
if (current != NULL) {
- const char *comment_end = tok->cur;
+ _PyTok_Off comment_end = tok->cur;
if (c == '\n' || c == '\r') {
comment_end--;
}
@@ -333,14 +323,14 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
}
if (tok->tok_extra_tokens) {
- p = tok->start;
+ p = _PyLexer_BufferPointer(tok, tok->start);
}
if (tok->type_comments) {
- p = tok->start;
+ p = _PyLexer_BufferPointer(tok, tok->start);
current_starting_col_offset = tok->start_loc.byte_col;
prefix = type_comment_prefix;
- while (*prefix && p < tok->cur) {
+ while (*prefix && p < _PyLexer_BufferPointer(tok, tok->cur)) {
if (*prefix == ' ') {
while (*p == ' ' || *p == '\t') {
p++;
@@ -369,8 +359,8 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
/* A TYPE_IGNORE is "type: ignore" followed by the end of the
token
* or anything ASCII and non-alphanumeric. */
is_type_ignore = (
- tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
- && !(tok->cur > ignore_end
+ _PyLexer_BufferPointer(tok, tok->cur) >= ignore_end &&
memcmp(p, "ignore", 6) == 0
+ && !(_PyLexer_BufferPointer(tok, tok->cur) > ignore_end
&& ((unsigned char)ignore_end[0] >= 128 ||
Py_ISALNUM(ignore_end[0]))));
int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
@@ -378,7 +368,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
? ignore_end_col_offset : current_starting_col_offset;
p_end = tok->cur;
if (is_type_ignore) {
- p_start = ignore_end;
+ p_start = _PyLexer_BufferOffset(tok, ignore_end);
/* If this type ignore is the only thing on the line,
consume the newline also. */
if (blankline) {
@@ -386,7 +376,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
tok->atbol = 1;
}
} else {
- p_start = type_start;
+ p_start = _PyLexer_BufferOffset(tok, type_start);
}
_PyLexer_token_setup(tok, token, type, p_start, p_end);
token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
@@ -397,7 +387,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
}
if (tok->tok_extra_tokens) {
tok_backup(tok, c); /* don't eat the newline or EOF */
- p_start = p;
+ p_start = _PyLexer_BufferOffset(tok, p);
p_end = tok->cur;
tok->comment_newline = blankline;
return MAKE_TOKEN(COMMENT);
@@ -668,9 +658,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
}
-int
+void
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
{
+ _PyToken_Free(token);
ftstring_state *current = _PyLexer_CurrentFTString(tok);
int result;
if (current == NULL) {
@@ -692,5 +683,5 @@ _PyTokenizer_Get(struct tok_state *tok, struct token *token)
if (tok_failed(tok)) {
result = ERRORTOKEN;
}
- return result;
+ token->type = result;
}
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 63f6e628640107c..7302198cc91010b 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -3,21 +3,4 @@
#include "state.h"
-int _PyTokenizer_Get(struct tok_state *, struct token *);
-
-/* The view points into the current input window. The next
- _PyTokenizer_Get() call may discard it. */
-static inline const char *
-_PyToken_TextView(const struct tok_state *tok, const struct token *token,
- Py_ssize_t *length)
-{
- assert(length != NULL);
- if (token->span.start < 0) {
- assert(token->span.start == -1 && token->span.end == -1);
- *length = 0;
- return "";
- }
- return _PyLexer_BufferSpanView(tok, token->span, length);
-}
-
#endif
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index 95d9754080bedbd..210d182d42dd9f5 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -21,7 +21,6 @@
(_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE)
#define FTSTRING_END(state) \
(_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END)
-#define tok_nextc _PyLexer_nextc
#define tok_backup _PyLexer_backup
static inline int
@@ -31,10 +30,29 @@ tok_failed(const struct tok_state *tok)
tok->done != E_INTERACT_STOP;
}
-int _PyLexer_nextc(struct tok_state *);
+int _PyLexer_refill(struct tok_state *);
+
+static inline int
+tok_nextc(struct tok_state *tok)
+{
+ while (tok->cur == tok->inp) {
+ if (!_PyLexer_refill(tok)) {
+ return EOF;
+ }
+ }
+ assert(tok->cur >= tok->source.base_offset);
+ assert(tok->cur - tok->source.base_offset < tok->source.len);
+ if (tok->cur - tok->line_start >= INT_MAX) {
+ tok->done = E_COLUMNOVERFLOW;
+ return EOF;
+ }
+ return Py_CHARMASK(
+ tok->source.bytes[tok->cur++ - tok->source.base_offset]);
+}
+
void _PyLexer_backup(struct tok_state *, int);
int _PyLexer_record_ftstring_comment(
- struct tok_state *, ftstring_state *, const char *, const char *);
+ struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off);
int _PyLexer_ftstring_punctuation(
struct tok_state *, ftstring_state *, struct token *, int);
int _PyLexer_close_ftstring_expr(
diff --git a/Parser/lexer/number.c b/Parser/lexer/number.c
index 8bca8cbb9adfe5e..220952a69cd69e8 100644
--- a/Parser/lexer/number.c
+++ b/Parser/lexer/number.c
@@ -113,8 +113,8 @@ int
_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
int leading_dot)
{
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
if (leading_dot) {
goto fraction;
@@ -214,7 +214,7 @@ _PyLexer_scan_number(struct tok_state *tok, struct token
*token, int c,
}
c = tok_nextc(tok);
}
- char* zeros_end = tok->cur;
+ _PyTok_Off zeros_end = tok->cur;
if (Py_ISDIGIT(c)) {
nonzero = 1;
c = tok_decimal_tail(tok);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index cdf997363b00038..a6617c33480c855 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -19,11 +19,12 @@ _PyTokenizer_tok_new(void)
return NULL;
}
- tok->buf = tok->cur = tok->inp = NULL;
+ tok->cur = tok->inp = 0;
+ tok->line_start = -1;
tok->fp_interactive = 0;
tok->interactive_src_start = NULL;
tok->interactive_src_end = NULL;
- tok->start = NULL;
+ tok->start = -1;
tok->done = E_OK;
tok->fp = NULL;
tok->indent = 0;
@@ -123,27 +124,24 @@ _PyTokenizer_Free(struct tok_state *tok)
PyMem_Free(tok);
}
-void
-_PyToken_Free(struct token *token) {
- Py_XDECREF(token->metadata);
-}
-
void
_PyToken_Init(struct token *token) {
-#ifdef Py_DEBUG
- token->span = (_PyTok_Span){-1, -1};
- token->start_loc = (_PyTok_Loc){-1, -1};
- token->end_loc = (_PyTok_Loc){-1, -1};
-#endif
- token->metadata = NULL;
+ *token = (struct token){
+ .type = -1,
+ .span = {-1, -1},
+ .start_loc = {-1, -1},
+ .end_loc = {-1, -1},
+ };
}
int
-_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type,
const char *start, const char *end)
+_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type,
_PyTok_Off start, _PyTok_Off end)
{
token->level = tok->level;
- token->span = _PyLexer_BufferSpan(tok, start, end);
- if (start != NULL && end != NULL) {
+ token->is_raw = 0;
+ assert((start == -1 && end == -1) || (start >= 0 && end >= start));
+ token->span = (_PyTok_Span){start, end};
+ if (start >= 0) {
token->start_loc = tok->start_loc;
token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
}
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 6203084f19b54f5..0f9ddb6d45e9611 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -3,6 +3,7 @@
#include "object.h"
#include "../tokenizer/source.h"
+#include "../tokenizer/tokenizer.h"
#define MAXINDENT 100 /* Max indentation level */
#define MAXLEVEL 200 /* Max parentheses level */
@@ -17,13 +18,6 @@ enum interactive_underflow_t {
IUNDERFLOW_STOP,
};
-struct token {
- int level;
- _PyTok_Span span;
- _PyTok_Loc start_loc;
- _PyTok_Loc end_loc;
- PyObject *metadata;
-};
typedef enum {
FTSTRING_MODE_MIDDLE,
@@ -74,16 +68,15 @@ _PyLexer_IsRawString(ftstring_kind kind)
/* Tokenizer state */
struct tok_state {
- /* Input state; buf <= cur <= inp */
- /* NB an entire line is held in the buffer */
- char *buf;
- char *cur; /* Next character in buffer */
- char *inp; /* End of data in buffer */
- _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
+ _PyTok_Off buf_offset;
+ _PyTok_Off cur;
+ _PyTok_Off inp;
+ _PyTok_Off start;
+ _PyTok_Off line_start;
+ _PyTok_SourceText source;
int fp_interactive; /* If the file descriptor is interactive */
char *interactive_src_start; /* The start of the source parsed so far in
interactive mode */
char *interactive_src_end; /* The end of the source parsed so far in
interactive mode */
- const char *start; /* Start of current token if not NULL */
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
@@ -105,10 +98,8 @@ struct tok_state {
int altindstack[MAXINDENT]; /* Stack of alternate indents */
/* Stuff for PEP 0263 */
char *encoding; /* Source encoding. */
- const char* line_start; /* pointer to start of current line */
char* str; /* Source string being tokenized (if tokenizing from a
string)*/
- _PyTok_SourceText source;
struct _PyTok_Reader *reader;
int type_comments; /* Whether to look for type comments */
@@ -155,22 +146,17 @@ _PyLexer_FTStringBracketDepth(const struct tok_state *tok,
static inline _PyTok_Off
_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
{
- assert(tok->buf != NULL);
- assert(tok->inp >= tok->buf);
- assert(position >= tok->buf && position <= tok->inp);
- Py_ssize_t offset = position - tok->buf;
- assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset);
- return tok->buf_offset + offset;
+ const char *base = _PyTok_SourceData(&tok->source);
+ assert(position >= base && position <= base + tok->source.len);
+ return tok->source.base_offset + (position - base);
}
-static inline char *
+static inline const char *
_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
{
- assert(tok->buf != NULL);
- assert(tok->inp >= tok->buf);
- assert(offset >= tok->buf_offset);
- assert(offset - tok->buf_offset <= tok->inp - tok->buf);
- return tok->buf + (offset - tok->buf_offset);
+ assert(offset >= tok->source.base_offset);
+ assert(offset - tok->source.base_offset <= tok->source.len);
+ return _PyTok_SourceData(&tok->source) + (offset -
tok->source.base_offset);
}
static inline const char *
@@ -187,36 +173,19 @@ _PyLexer_BufferSpanView(const struct tok_state *tok,
_PyTok_Span span,
static inline int
_PyLexer_ByteColumn(const struct tok_state *tok)
{
- assert(tok->line_start != NULL);
+ assert(tok->line_start >= 0);
assert(tok->cur >= tok->line_start);
Py_ssize_t column = tok->cur - tok->line_start;
assert(column <= INT_MAX);
return (int)column;
}
-static inline _PyTok_Span
-_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
- const char *end)
-{
- if (start == NULL) {
- assert(end == NULL);
- return (_PyTok_Span){-1, -1};
- }
- assert(end != NULL);
- assert(start <= end);
- return _PyTok_SpanFromBounds(
- _PyLexer_BufferOffset(tok, start),
- _PyLexer_BufferOffset(tok, end));
-}
-
-int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type,
const char *start, const char *end);
+int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type,
_PyTok_Off start, _PyTok_Off end);
struct tok_state *_PyTokenizer_tok_new(void);
void _PyTokenizer_Free(struct tok_state *);
ftstring_state *_PyLexer_PushFTString(struct tok_state *);
void _PyLexer_PopFTString(struct tok_state *);
-void _PyToken_Free(struct token *);
-void _PyToken_Init(struct token *);
#endif
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index ac775e9d61ba096..945480ef86f7789 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -8,17 +8,17 @@
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type,
p_start, p_end)
static void
-rewind_to_string_start(struct tok_state *tok, const char *start,
+rewind_to_string_start(struct tok_state *tok, _PyTok_Off start,
_PyTok_Loc location)
{
- tok->cur = (char *)start + 1;
+ tok->cur = start + 1;
tok->line_start = start - location.byte_col;
tok->lineno = location.lineno;
}
int
_PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
- const char *start, const char *end)
+ _PyTok_Off start, _PyTok_Off end)
{
assert(state == _PyLexer_CurrentFTString(tok) && state->mode ==
FTSTRING_MODE_EXPRESSION);
if (state->expr_span.end >= 0) {
@@ -50,7 +50,7 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok,
ftstring_state *state,
state->comments = comments;
}
comments->spans[comments->count++] =
- _PyLexer_BufferSpan(tok, start, end);
+ (_PyTok_Span){start, end};
return 0;
}
@@ -59,13 +59,13 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state
*state,
struct token *token)
{
assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
- assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL);
+ assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start >= 0);
if (state->expr_span.end >= 0) {
return 0;
}
assert(state->expr_span.start >= 0);
- state->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
+ state->expr_span.end = tok->start;
int tstring_interpolation = _PyLexer_IsTString(state->kind) &&
state->replacement_depth == 1;
if (!(state->debug_expr || tstring_interpolation) || token->metadata) {
@@ -251,8 +251,8 @@ _PyLexer_check_string_prefixes(struct tok_state *tok,
int
_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
{
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
int quote = c;
int quote_size = 1; /* 1 or 3 */
@@ -284,26 +284,26 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct
token *token, int c)
state->quote = quote;
state->quote_size = quote_size;
state->paren_level = tok->level;
- state->start = _PyLexer_BufferOffset(tok, tok->start);
+ state->start = tok->start;
state->start_loc = tok->start_loc;
state->expr_span = (_PyTok_Span){-1, -1};
int raw = 0;
int tstring = 0;
- switch (*tok->start) {
+ switch (*_PyLexer_BufferPointer(tok, tok->start)) {
case 'T':
case 't':
- raw = Py_TOLOWER(tok->start[1]) == 'r';
+ raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) ==
'r';
tstring = 1;
break;
case 'F':
case 'f':
- raw = Py_TOLOWER(tok->start[1]) == 'r';
+ raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) ==
'r';
break;
case 'R':
case 'r':
raw = 1;
- tstring = Py_TOLOWER(tok->start[1]) == 't';
+ tstring = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1])
== 't';
break;
default:
Py_UNREACHABLE();
@@ -317,8 +317,8 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct
token *token, int c)
int
_PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
{
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
int quote = c;
int quote_size = 1; /* 1 or 3 */
@@ -419,8 +419,9 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state
*current, struct tok
assert(current == _PyLexer_CurrentFTString(tok) && current->mode !=
FTSTRING_MODE_EXPRESSION);
assert((current->quote_size == 1 || current->quote_size == 3) &&
current->replacement_depth <= MAX_EXPR_NESTING);
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
+ int token_type;
int end_quote_size = 0;
int unicode_escape = 0;
int quote = current->quote;
@@ -452,7 +453,7 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state
*current, struct tok
int end_lineno = tok->lineno;
rewind_to_string_start(tok,
- _PyLexer_BufferPointer(tok, current->start),
+ current->start,
current->start_loc);
if (quote_size == 3) {
@@ -484,7 +485,7 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state
*current, struct tok
int peek = tok_nextc(tok);
if (peek != '{' || in_format_spec) {
tok_backup(tok, peek);
- _PyTok_Off expr_start = _PyLexer_BufferOffset(tok, tok->cur);
+ _PyTok_Off expr_start = tok->cur;
tok_backup(tok, c);
if (begin_ftstring_expr(tok, current, expr_start) < 0) {
return MAKE_TOKEN(ERRORTOKEN);
@@ -572,5 +573,7 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state
*current, struct tok
}
p_end = tok->cur;
emit_middle:
- return MAKE_TOKEN(FTSTRING_MIDDLE(current));
+ token_type = MAKE_TOKEN(FTSTRING_MIDDLE(current));
+ token->is_raw = raw;
+ return token_type;
}
diff --git a/Parser/pegen.c b/Parser/pegen.c
index 20dbbcabec379e9..e709031ae781598 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -7,9 +7,7 @@
#include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal
#include <errcode.h>
-#include "lexer/lexer.h"
#include "tokenizer/tokenizer.h"
-#include "tokenizer/helpers.h"
#include "pegen.h"
#define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two.
@@ -208,7 +206,6 @@ initialize_token(Parser *p, Token *parser_token, struct
token *new_token, int to
parser_token->metadata = NULL;
if (new_token->metadata != NULL) {
if (_PyArena_AddPyObject(p->arena, new_token->metadata) < 0) {
- Py_DECREF(new_token->metadata);
return -1;
}
parser_token->metadata = new_token->metadata;
@@ -216,18 +213,19 @@ initialize_token(Parser *p, Token *parser_token, struct
token *new_token, int to
}
parser_token->level = new_token->level;
+ parser_token->is_raw = new_token->is_raw;
parser_token->lineno = new_token->start_loc.lineno;
- parser_token->col_offset = p->tok->lineno == p->starting_lineno
+ parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
? p->starting_col_offset + new_token->start_loc.byte_col
: new_token->start_loc.byte_col;
parser_token->end_lineno = new_token->end_loc.lineno;
- parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+ parser_token->end_col_offset = new_token->end_loc.lineno ==
p->starting_lineno
? p->starting_col_offset + new_token->end_loc.byte_col
: new_token->end_loc.byte_col;
p->fill += 1;
- if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
+ if (token_type == ERRORTOKEN && _PyTokenizer_GetInfo(p->tok).status ==
E_DECODE) {
return _Pypegen_raise_decode_error(p);
}
@@ -261,10 +259,10 @@ _PyPegen_fill_token(Parser *p)
{
struct token new_token;
_PyToken_Init(&new_token);
- int type = _PyTokenizer_Get(p->tok, &new_token);
+ _PyTokenizer_Get(p->tok, &new_token);
// Record and skip '# type: ignore' comments
- while (type == TYPE_IGNORE) {
+ while (new_token.type == TYPE_IGNORE) {
Py_ssize_t len;
const char *text = _PyToken_TextView(p->tok, &new_token, &len);
char *tag = PyMem_Malloc((size_t)len + 1);
@@ -275,21 +273,22 @@ _PyPegen_fill_token(Parser *p)
memcpy(tag, text, (size_t)len);
tag[len] = '\0';
// Ownership of tag passes to the growable array
- if (!growable_comment_array_add(&p->type_ignore_comments,
p->tok->lineno, tag)) {
+ if (!growable_comment_array_add(&p->type_ignore_comments,
new_token.end_loc.lineno, tag)) {
PyErr_NoMemory();
goto error;
}
- type = _PyTokenizer_Get(p->tok, &new_token);
+ _PyTokenizer_Get(p->tok, &new_token);
}
+ int type = new_token.type;
+
// If we have reached the end and we are in single input mode we need to
insert a newline and reset the parsing
if (p->start_rule == Py_single_input && type == ENDMARKER &&
p->parsing_started) {
type = NEWLINE; /* Add an extra newline */
p->parsing_started = 0;
- if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
- p->tok->pendin = -p->tok->indent;
- p->tok->indent = 0;
+ if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
+ _PyTokenizer_ImplyDedents(p->tok);
}
}
else {
@@ -302,7 +301,9 @@ _PyPegen_fill_token(Parser *p)
}
Token *t = p->tokens[p->fill];
- return initialize_token(p, t, &new_token, type);
+ int result = initialize_token(p, t, &new_token, type);
+ _PyToken_Free(&new_token);
+ return result;
error:
_PyToken_Free(&new_token);
return -1;
@@ -793,34 +794,7 @@ _PyPegen_number_token(Parser *p)
t->end_col_offset, p->arena);
}
-/* Check that the source for a single input statement really is a single
- statement by looking at what is left in the buffer after parsing.
- Trailing whitespace and comments are OK. */
-static int // bool
-bad_single_statement(Parser *p)
-{
- char *cur = p->tok->cur;
- char c = *cur;
-
- for (;;) {
- while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
- c = *++cur;
- }
-
- if (!c) {
- return 0;
- }
-
- if (c != '#') {
- return 1;
- }
- /* Suck up comment. */
- while (c && c != '\n') {
- c = *++cur;
- }
- }
-}
static int
compute_parser_flags(PyCompilerFlags *flags)
@@ -858,7 +832,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule,
int flags,
return (Parser *) PyErr_NoMemory();
}
assert(tok != NULL);
- tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
+ _PyTokenizer_SetOptions(tok, 0, (flags & PyPARSE_TYPE_COMMENTS) > 0);
p->tok = tok;
p->keywords = NULL;
p->n_keyword_lists = -1;
@@ -943,14 +917,12 @@ reset_parser_state_for_error_pass(Parser *p)
}
p->mark = 0;
p->call_invalid_rules = 1;
- // Don't try to get extra tokens in interactive mode when trying to
- // raise specialized errors in the second pass.
- p->tok->interactive_underflow = IUNDERFLOW_STOP;
+ _PyTokenizer_StopInteractive(p->tok);
}
static inline int
_is_end_of_source(Parser *p) {
- int err = p->tok->done;
+ int err = _PyTokenizer_GetInfo(p->tok).status;
return err == E_EOF || err == E_EOFS || err == E_EOLS;
}
@@ -961,19 +933,14 @@ _PyPegen_set_syntax_error_metadata(Parser *p) {
PyErr_SetRaisedException(exc);
return;
}
- const char *source = NULL;
- if (p->tok->str != NULL) {
- source = p->tok->str;
- }
- if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) {
- source = p->tok->interactive_src_start;
- }
+ const char *source = _PyTokenizer_RetainedSource(p->tok);
+ const char *encoding = _PyTokenizer_GetInfo(p->tok).encoding;
PyObject* the_source = NULL;
if (source) {
- if (p->tok->encoding == NULL) {
+ if (encoding == NULL) {
the_source = PyUnicode_FromString(source);
} else {
- the_source = PyUnicode_Decode(source, strlen(source),
p->tok->encoding, NULL);
+ the_source = PyUnicode_Decode(source, strlen(source), encoding,
NULL);
}
}
if (!the_source) {
@@ -1034,7 +1001,7 @@ _PyPegen_run_parser(Parser *p)
return NULL;
}
- if (p->start_rule == Py_single_input && bad_single_statement(p)) {
+ if (p->start_rule == Py_single_input &&
_PyTokenizer_HasTrailingStatement(p->tok)) {
return RAISE_SYNTAX_ERROR("multiple statements found while compiling a
single statement");
}
@@ -1069,20 +1036,16 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int
start_rule, PyObject *filena
}
return NULL;
}
- if (!tok->fp || ps1 != NULL || ps2 != NULL ||
- PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
- tok->fp_interactive = 1;
- }
- // This transfers the ownership to the tokenizer
- tok->filename = Py_NewRef(filename_ob);
// From here on we need to clean up even if there's an error
mod_ty result = NULL;
- tok->module = PyUnicode_FromString("__main__");
- if (tok->module == NULL) {
+ PyObject *module = PyUnicode_FromString("__main__");
+ if (module == NULL) {
goto error;
}
+ _PyTokenizer_SetContext(tok, filename_ob, module);
+ Py_DECREF(module);
int parser_flags = compute_parser_flags(flags);
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags,
PY_MINOR_VERSION,
@@ -1094,8 +1057,9 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int
start_rule, PyObject *filena
result = _PyPegen_run_parser(p);
_PyPegen_Parser_Free(p);
- if (tok->fp_interactive && tok->interactive_src_start && result &&
interactive_src != NULL) {
- *interactive_src = PyUnicode_FromString(tok->interactive_src_start);
+ const char *source = _PyTokenizer_RetainedSource(tok);
+ if (source != NULL && result && interactive_src != NULL) {
+ *interactive_src = PyUnicode_FromString(source);
if (!*interactive_src || _PyArena_AddPyObject(arena, *interactive_src)
< 0) {
Py_XDECREF(*interactive_src);
result = NULL;
@@ -1130,9 +1094,7 @@ _PyPegen_run_parser_from_string(const char *str, int
start_rule, PyObject *filen
}
return NULL;
}
- // This transfers the ownership to the tokenizer
- tok->filename = Py_NewRef(filename_ob);
- tok->module = Py_XNewRef(module);
+ _PyTokenizer_SetContext(tok, filename_ob, module);
// We need to clear up from here on
mod_ty result = NULL;
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 81137782621daf4..5ebf6aded852787 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -5,7 +5,7 @@
#include <pycore_ast.h>
#include <pycore_token.h>
-#include "lexer/state.h"
+#include "tokenizer/tokenizer.h"
#if 0
#define PyPARSE_YIELD_IS_KEYWORD 0x0001
@@ -37,6 +37,7 @@ typedef struct {
int type;
PyObject *bytes;
int level;
+ int is_raw;
int lineno, col_offset, end_lineno, end_col_offset;
Memo *memo;
// Filter over the rule types present in `memo` (bit `type & 63` is set
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index f4c3f988706259d..7841f01b612915c 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -4,20 +4,20 @@
#include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject()
#include "pycore_runtime.h" // _Py_ID()
#include "pycore_tuple.h" // _PyTuple_FromPair
-#include "lexer/state.h"
-#include "lexer/lexer.h"
#include "pegen.h"
+#include "tokenizer/tokenizer.h"
// TOKENIZER ERRORS
static inline void
raise_unclosed_parentheses_error(Parser *p) {
- int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
- int error_col = p->tok->parencolstack[p->tok->level-1];
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ int error_lineno = info.delimiter_loc.lineno;
+ int error_col = info.delimiter_loc.byte_col;
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError,
error_lineno, error_col, error_lineno, -1,
"'%c' was never closed",
- p->tok->parenstack[p->tok->level-1]);
+ info.delimiter);
}
int
@@ -31,12 +31,13 @@ _Pypegen_tokenizer_error(Parser *p)
PyObject* errtype = PyExc_SyntaxError;
Py_ssize_t col_offset = -1;
p->error_indicator = 1;
- switch (p->tok->done) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ switch (info.status) {
case E_TOKEN:
msg = "invalid token";
break;
case E_EOF:
- if (p->tok->level) {
+ if (info.level) {
raise_unclosed_parentheses_error(p);
} else {
RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
@@ -62,7 +63,7 @@ _Pypegen_tokenizer_error(Parser *p)
msg = "too many levels of indentation";
break;
case E_LINECONT: {
- col_offset = p->tok->cur - p->tok->line_start - 1;
+ col_offset = info.cursor - info.line_span.start - 1;
msg = "unexpected character after line continuation character";
break;
}
@@ -74,9 +75,9 @@ _Pypegen_tokenizer_error(Parser *p)
msg = "unknown parsing error";
}
- RAISE_ERROR_KNOWN_LOCATION(p, errtype, p->tok->lineno,
+ RAISE_ERROR_KNOWN_LOCATION(p, errtype, info.location.lineno,
col_offset >= 0 ? col_offset : 0,
- p->tok->lineno, -1, msg);
+ info.location.lineno, -1, msg);
return -1;
}
@@ -122,7 +123,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser
*p) {
// before the one that we had for the generic error.
// We don't want to tokenize to the end for interactive input
- if (p->tok->prompt != NULL) {
+ if (_PyTokenizer_IsInteractive(p->tok)) {
return 0;
}
@@ -137,14 +138,16 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser
*p) {
_PyToken_Init(&new_token);
for (;;) {
- switch (_PyTokenizer_Get(p->tok, &new_token)) {
- case ERRORTOKEN:
+ _PyTokenizer_Get(p->tok, &new_token);
+ switch (new_token.type) {
+ case ERRORTOKEN: {
if (PyErr_Occurred()) {
ret = -1;
goto exit;
}
- if (p->tok->level != 0) {
- int error_lineno =
p->tok->parenlinenostack[p->tok->level-1];
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ if (info.level != 0) {
+ int error_lineno = info.delimiter_loc.lineno;
if (current_err_line > error_lineno) {
raise_unclosed_parentheses_error(p);
ret = -1;
@@ -152,6 +155,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser
*p) {
}
}
break;
+ }
case ENDMARKER:
break;
default:
@@ -163,8 +167,9 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser
*p) {
exit:
_PyToken_Free(&new_token);
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
// Preserve expression errors over later formatted-string errors.
- if (PyErr_Occurred() && _PyLexer_CurrentFTString(p->tok) == NULL) {
+ if (PyErr_Occurred() && !info.in_formatted_string) {
Py_XDECREF(value);
Py_XDECREF(type);
Py_XDECREF(traceback);
@@ -200,11 +205,12 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int
use_mark, const char *err
Py_ssize_t col_offset;
Py_ssize_t end_col_offset = -1;
if (t->col_offset == -1) {
- if (p->tok->cur == p->tok->buf) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ if (info.cursor == info.input_span.start) {
col_offset = 0;
} else {
- const char* start = p->tok->buf ? p->tok->line_start :
p->tok->buf;
- col_offset = Py_SAFE_DOWNCAST(p->tok->cur - start, intptr_t, int);
+ col_offset = Py_SAFE_DOWNCAST(
+ info.cursor - info.line_span.start, intptr_t, int);
}
} else {
col_offset = t->col_offset + 1;
@@ -223,46 +229,15 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int
use_mark, const char *err
}
static PyObject *
-get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
+get_error_line_from_source(Parser *p, Py_ssize_t lineno)
{
- /* If the file descriptor is interactive, the source lines of the current
- * (multi-line) statement are stored in p->tok->interactive_src_start.
- * If not, we're parsing from a string, which means that the whole source
- * is stored in p->tok->str. */
- assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL);
-
- char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start :
p->tok->str;
- if (cur_line == NULL) {
- assert(p->tok->fp_interactive);
- // We can reach this point if the tokenizer buffers for interactive
source have not been
- // initialized because we failed to decode the original source with
the given locale.
+ if (_PyTokenizer_RetainedSource(p->tok) == NULL) {
return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
-
Py_ssize_t relative_lineno = p->starting_lineno ? lineno -
p->starting_lineno + 1 : lineno;
- const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end
: p->tok->inp;
-
- if (buf_end < cur_line) {
- buf_end = cur_line + strlen(cur_line);
- }
-
- for (int i = 0; i < relative_lineno - 1; i++) {
- char *new_line = strchr(cur_line, '\n');
- // The assert is here for debug builds but the conditional that
- // follows is there so in release builds we do not crash at the cost
- // to report a potentially wrong line.
- assert(new_line != NULL && new_line + 1 < buf_end);
- if (new_line == NULL || new_line + 1 > buf_end) {
- break;
- }
- cur_line = new_line + 1;
- }
-
- char *next_newline;
- if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last
line
- next_newline = cur_line + strlen(cur_line);
- }
- return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
+ Py_ssize_t len;
+ const char *line = _PyTokenizer_LineView(p->tok, relative_lineno, &len);
+ return PyUnicode_DecodeUTF8(line, len, "replace");
}
void *
@@ -280,12 +255,15 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject
*errtype,
PyObject *error_line = NULL;
PyObject *tmp = NULL;
p->error_indicator = 1;
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ _PyTok_Loc location = info.location;
+ _PyTok_Span text_span = info.line_span;
if (end_lineno == CURRENT_POS) {
- end_lineno = p->tok->lineno;
+ end_lineno = location.lineno;
}
if (end_col_offset == CURRENT_POS) {
- end_col_offset = p->tok->cur - p->tok->line_start;
+ end_col_offset = location.byte_col;
}
errstr = PyUnicode_FromFormatV(errmsg, va);
@@ -293,12 +271,12 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject
*errtype,
goto error;
}
- if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ if (info.is_interactive && _PyTokenizer_RetainedSource(p->tok) != NULL) {
+ error_line = get_error_line_from_source(p, lineno);
}
else if (p->start_rule == Py_file_input) {
- error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename,
- (int) lineno,
p->tok->encoding);
+ error_line = _PyErr_ProgramDecodedTextObject(info.filename,
+ (int) lineno,
info.encoding);
}
if (!error_line) {
@@ -309,14 +287,17 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject
*errtype,
we're actually parsing from a file, which has an E_EOF SyntaxError
and in that case
`PyErr_ProgramTextObject` fails because lineno points to
last_file_line + 1, which
does not physically exist */
- assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done ==
E_EOF);
-
- if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
- Py_ssize_t size = p->tok->inp - p->tok->line_start;
- error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size,
"replace");
+ assert(!info.is_file || info.status == E_EOF);
+
+ if (location.lineno <= lineno &&
+ info.input_span.end > info.input_span.start) {
+ Py_ssize_t size;
+ const char *line = _PyTokenizer_SpanView(
+ p->tok, text_span, &size);
+ error_line = PyUnicode_DecodeUTF8(line, size, "replace");
}
- else if (p->tok->fp == NULL || p->tok->fp == stdin) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ else if (!info.is_file) {
+ error_line = get_error_line_from_source(p, lineno);
}
else {
error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
@@ -341,7 +322,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject
*errtype,
}
}
- tmp = Py_BuildValue("(OnnNnn)", p->tok->filename, lineno, col_number,
error_line, end_lineno, end_col_number);
+ tmp = Py_BuildValue("(OnnNnn)", info.filename, lineno, col_number,
error_line, end_lineno, end_col_number);
if (!tmp) {
goto error;
}
@@ -364,11 +345,12 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject
*errtype,
void
_Pypegen_set_syntax_error(Parser* p, Token* last_token) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
// Existing syntax error
if (PyErr_Occurred()) {
// Prioritize tokenizer errors to custom syntax errors raised
// on the second phase only if the errors come from the parser.
- int is_tok_ok = (p->tok->done == E_DONE || p->tok->done == E_OK);
+ int is_tok_ok = (info.status == E_DONE || info.status == E_OK);
if (is_tok_ok && PyErr_ExceptionMatches(PyExc_SyntaxError)) {
_PyPegen_tokenize_full_source_to_check_for_errors(p);
}
@@ -380,8 +362,8 @@ _Pypegen_set_syntax_error(Parser* p, Token* last_token) {
RAISE_SYNTAX_ERROR("error at start before reading any input");
}
// Parser encountered EOF (End of File) unexpectedtly
- if (last_token->type == ERRORTOKEN && p->tok->done == E_EOF) {
- if (p->tok->level) {
+ if (last_token->type == ERRORTOKEN && info.status == E_EOF) {
+ if (info.level) {
raise_unclosed_parentheses_error(p);
} else {
RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
diff --git a/Parser/string_parser.c b/Parser/string_parser.c
index 431d2703d88e2c4..e57460cb1fa6483 100644
--- a/Parser/string_parser.c
+++ b/Parser/string_parser.c
@@ -2,7 +2,6 @@
#include "pycore_bytesobject.h" // _PyBytes_DecodeEscape()
#include "pycore_unicodeobject.h" // _PyUnicode_DecodeUnicodeEscapeInternal()
-#include "lexer/state.h"
#include "pegen.h"
#include "string_parser.h"
@@ -87,8 +86,9 @@ warn_invalid_escape_sequence(Parser *p, const char* buffer,
const char *first_in
col_offset += quote_count;
}
- if (PyErr_WarnExplicitObject(category, msg, p->tok->filename,
- lineno, p->tok->module, NULL) < 0) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ if (PyErr_WarnExplicitObject(category, msg, info.filename,
+ lineno, info.module, NULL) < 0) {
if (PyErr_ExceptionMatches(category)) {
/* Replace the Syntax/DeprecationWarning exception with a
SyntaxError
to get a more accurate error report */
diff --git a/Parser/tokenizer/api.c b/Parser/tokenizer/api.c
new file mode 100644
index 000000000000000..67e129d52441c76
--- /dev/null
+++ b/Parser/tokenizer/api.c
@@ -0,0 +1,170 @@
+#include "Python.h"
+#include "errcode.h"
+#include "pycore_token.h"
+
+#include "tokenizer.h"
+#include "reader.h"
+#include "reader_internal.h"
+#include "../lexer/state.h"
+
+_PyTokenizer_Info
+_PyTokenizer_GetInfo(const struct tok_state *tok)
+{
+ _PyTokenizer_Info info = {
+ .status = tok->done,
+ .location = {tok->lineno, tok->line_start < 0
+ ? -1 : (int)(tok->cur - tok->line_start)},
+ .cursor = tok->cur,
+ .input_span = {tok->buf_offset, tok->inp},
+ .line_span = {tok->line_start, tok->inp},
+ .level = tok->level,
+ .delimiter_loc = {-1, -1},
+ .in_formatted_string = tok->ftstring_depth != 0,
+ .is_interactive = tok->reader->kind == _PYTOK_READER_INTERACTIVE,
+ .is_file = tok->fp != NULL && tok->fp != stdin,
+ .filename = tok->filename,
+ .module = tok->module,
+ .encoding = tok->encoding,
+ };
+ if (tok->level > 0) {
+ int level = tok->level - 1;
+ info.delimiter = tok->parenstack[level];
+ info.delimiter_loc = (_PyTok_Loc){
+ tok->parenlinenostack[level], tok->parencolstack[level]};
+ }
+ return info;
+}
+
+const char *
+_PyToken_TextView(const struct tok_state *tok, const struct token *token,
+ Py_ssize_t *length)
+{
+ assert(length != NULL);
+ if (token->span.start < 0) {
+ assert(token->span.start == -1 && token->span.end == -1);
+ *length = 0;
+ return "";
+ }
+ return _PyLexer_BufferSpanView(tok, token->span, length);
+}
+
+const char *
+_PyTokenizer_SpanView(const struct tok_state *tok, _PyTok_Span span,
+ Py_ssize_t *length)
+{
+ return _PyLexer_BufferSpanView(tok, span, length);
+}
+
+void
+_PyToken_GetView(const struct tok_state *tok, const struct token *token,
+ _PyToken_View *view)
+{
+ assert(view != NULL);
+ assert((token->span.start == -1 && token->span.end == -1) ||
+ _PyTok_SpanIsValid(token->span));
+ if (token->span.start >= 0) {
+ (void)_PyLexer_BufferPointer(tok, token->span.end);
+ }
+ view->text = token->span.start < 0
+ ? NULL : _PyLexer_BufferPointer(tok, token->span.start);
+ view->length = token->span.end - token->span.start;
+ view->end_line = _PyLexer_BufferPointer(tok, tok->line_start);
+ view->line = ISSTRINGLIT(token->type)
+ ? view->text - token->start_loc.byte_col : view->end_line;
+ view->line_length = tok->inp - tok->line_start +
+ (view->end_line - view->line);
+ view->implicit_newline = tok->implicit_newline;
+ view->at_eof = tok->done == E_EOF;
+}
+
+const char *
+_PyTokenizer_LineView(const struct tok_state *tok, Py_ssize_t lineno,
+ Py_ssize_t *length)
+{
+ const char *line = _PyTokenizer_RetainedSource(tok);
+ if (line == NULL) {
+ line = _PyLexer_BufferPointer(tok, tok->buf_offset);
+ }
+ for (Py_ssize_t i = 1; i < lineno; i++) {
+ const char *next = strchr(line, '\n');
+ if (next == NULL) {
+ break;
+ }
+ line = next + 1;
+ }
+ const char *end = strchr(line, '\n');
+ *length = end != NULL ? end - line : (Py_ssize_t)strlen(line);
+ return line;
+}
+
+const char *
+_PyTokenizer_RetainedSource(const struct tok_state *tok)
+{
+ if (tok->reader->kind == _PYTOK_READER_PREPARED) {
+ return _PyTok_SourceData(&tok->source);
+ }
+ if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
+ return tok->source.bytes;
+ }
+ return NULL;
+}
+
+void
+_PyTokenizer_SetContext(struct tok_state *tok, PyObject *filename,
+ PyObject *module)
+{
+ Py_XINCREF(filename);
+ Py_XINCREF(module);
+ Py_XSETREF(tok->filename, filename);
+ Py_XSETREF(tok->module, module);
+}
+
+void
+_PyTokenizer_SetOptions(struct tok_state *tok, int extra_tokens,
+ int type_comments)
+{
+ tok->tok_extra_tokens = extra_tokens;
+ tok->type_comments = type_comments;
+}
+
+void
+_PyTokenizer_ImplyDedents(struct tok_state *tok)
+{
+ if (tok->indent != 0) {
+ tok->pendin = -tok->indent;
+ tok->indent = 0;
+ }
+}
+
+int
+_PyTokenizer_HasTrailingStatement(const struct tok_state *tok)
+{
+ const char *cur = _PyLexer_BufferPointer(tok, tok->cur);
+ char c = *cur;
+ for (;;) {
+ while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
+ c = *++cur;
+ }
+ if (!c) {
+ return 0;
+ }
+ if (c != '#') {
+ return 1;
+ }
+ while (c && c != '\n') {
+ c = *++cur;
+ }
+ }
+}
+
+int
+_PyTokenizer_IsInteractive(const struct tok_state *tok)
+{
+ return tok->prompt != NULL;
+}
+
+void
+_PyTokenizer_StopInteractive(struct tok_state *tok)
+{
+ tok->interactive_underflow = IUNDERFLOW_STOP;
+}
diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c
index af17b8b63235f51..5b588572f1fb7e8 100644
--- a/Parser/tokenizer/decoder.c
+++ b/Parser/tokenizer/decoder.c
@@ -238,22 +238,13 @@ _PyTok_DetectEncoding(struct tok_state *tok, const
_PyTok_Chunk *first,
const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
const char *line_data = line->data + (cookie_line == 1 ? 3 : 0);
Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0);
- const char *saved_line_start = tok->line_start;
- char *saved_cur = tok->cur;
- int saved_lineno = tok->lineno;
- tok->line_start = line_data;
- tok->cur = (char *)line_data;
- tok->lineno = cookie_line;
int end_col = (int)Py_MIN(line_len, INT_MAX);
if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
line_data[end_col - 1] == '\r')) {
end_col--;
}
- _PyTokenizer_syntaxerror_known_range(
- tok, 0, end_col, "encoding problem: %s with BOM", cookie);
- tok->line_start = saved_line_start;
- tok->cur = saved_cur;
- tok->lineno = saved_lineno;
+ _PyTokenizer_syntaxerror_at(
+ tok, line_data, 0, cookie_line, 0, end_col, "encoding problem: %s
with BOM", cookie);
PyMem_Free(cookie);
return _PYTOK_ENCODING_ERROR;
}
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index e99be70ea5e9801..c803b787d9dae68 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -24,7 +24,8 @@ byte_col_to_char_col(const char *line, int byte_col)
}
static int
-_syntaxerror_range(struct tok_state *tok, const char *format,
+_syntaxerror_range(struct tok_state *tok, const char *line,
+ Py_ssize_t cursor, int lineno, const char *format,
int col_offset, int end_col_offset,
va_list vargs)
{
@@ -40,7 +41,7 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
goto error;
}
- errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start,
+ errtext = PyUnicode_DecodeUTF8(line, cursor,
"replace");
if (!errtext) {
goto error;
@@ -50,19 +51,19 @@ _syntaxerror_range(struct tok_state *tok, const char
*format,
col_offset = (int)PyUnicode_GET_LENGTH(errtext);
}
else if (col_offset > 0) {
- col_offset = byte_col_to_char_col(tok->line_start, col_offset);
+ col_offset = byte_col_to_char_col(line, col_offset);
}
if (end_col_offset == -1) {
end_col_offset = col_offset;
}
else if (end_col_offset > 0) {
- end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset);
+ end_col_offset = byte_col_to_char_col(line, end_col_offset);
}
- Py_ssize_t line_len = strcspn(tok->line_start, "\n");
- if (line_len != tok->cur - tok->line_start) {
+ Py_ssize_t line_len = strcspn(line, "\n");
+ if (line_len != cursor) {
Py_DECREF(errtext);
- errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len,
+ errtext = PyUnicode_DecodeUTF8(line, line_len,
"replace");
}
if (!errtext) {
@@ -71,8 +72,8 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
args = Py_BuildValue("(O(OiiNii))", errmsg,
tok->filename ? tok->filename : Py_None,
- tok->lineno, col_offset, errtext,
- tok->lineno, end_col_offset);
+ lineno, col_offset, errtext,
+ lineno, end_col_offset);
if (args) {
PyErr_SetObject(PyExc_SyntaxError, args);
Py_DECREF(args);
@@ -90,7 +91,9 @@ _PyTokenizer_syntaxerror(struct tok_state *tok, const char
*format, ...)
// These errors are cleaned on startup. Todo: Fix it.
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, format, -1, -1, vargs);
+ int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok,
tok->line_start),
+ tok->cur - tok->line_start, tok->lineno,
+ format, -1, -1, vargs);
va_end(vargs);
return ret;
}
@@ -102,7 +105,23 @@ _PyTokenizer_syntaxerror_known_range(struct tok_state *tok,
{
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset,
vargs);
+ int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok,
tok->line_start),
+ tok->cur - tok->line_start, tok->lineno,
+ format, col_offset, end_col_offset, vargs);
+ va_end(vargs);
+ return ret;
+}
+
+int
+_PyTokenizer_syntaxerror_at(struct tok_state *tok, const char *line,
+ Py_ssize_t cursor, int lineno,
+ int col_offset, int end_col_offset,
+ const char *format, ...)
+{
+ va_list vargs;
+ va_start(vargs, format);
+ int ret = _syntaxerror_range(tok, line, cursor, lineno, format,
+ col_offset, end_col_offset, vargs);
va_end(vargs);
return ret;
}
@@ -326,8 +345,8 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state
*tok, int lineno)
}
if (badchar) {
tok->lineno = lineno;
- tok->line_start = line_start;
- tok->cur = (char *)badchar;
+ tok->line_start = _PyLexer_BufferOffset(tok, line_start);
+ tok->cur = _PyLexer_BufferOffset(tok, badchar);
_PyTokenizer_syntaxerror_known_range(tok,
col_offset + 1, col_offset + 1,
"Non-UTF-8 code starting with '\\x%.2x'"
diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h
index 51de0cbb156f833..24f2d0cff1effb6 100644
--- a/Parser/tokenizer/helpers.h
+++ b/Parser/tokenizer/helpers.h
@@ -5,6 +5,8 @@
#include "../lexer/state.h"
+int _PyTokenizer_syntaxerror_at(struct tok_state *, const char *,
+ Py_ssize_t, int, int, int, const char *, ...);
int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int
col_offset, int end_col_offset, const char *format, ...);
int _PyTokenizer_indenterror(struct tok_state *tok);
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 6eb528ca985ed61..98a464210146891 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -5,7 +5,6 @@
#include "helpers.h"
#include "reader.h"
#include "reader_internal.h"
-#include "../lexer/buffer.h"
#include "../lexer/lexer.h"
#include "../lexer/state.h"
@@ -34,7 +33,6 @@ _PyTok_ReaderFree(struct tok_state *tok)
}
PyMem_Free(reader->file_buffer);
PyMem_Free(reader->decoded);
- tok->buf = NULL;
PyMem_Free(reader);
tok->reader = NULL;
}
@@ -142,7 +140,7 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
if (lineno > tok->source.nlines) {
return _PYTOK_READ_EOF;
}
- const char *start = tok->inp;
+ const char *start = _PyLexer_BufferPointer(tok, tok->inp);
const char *newline = memchr(
start, '\n', tok->source.bytes + tok->source.len - start);
_PyTok_Off end = newline != NULL
@@ -536,23 +534,21 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
static void
reset_streaming_buffer(struct tok_state *tok)
{
- assert(tok->buf != NULL);
- assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+ assert(tok->cur >= tok->buf_offset && tok->cur <= tok->inp);
_PyTok_SourceDiscard(&tok->source);
tok->buf_offset = tok->source.base_offset;
- tok->buf = tok->cur = tok->inp = (char *)_PyTok_SourceData(&tok->source);
- tok->line_start = tok->buf;
+ tok->cur = tok->inp = tok->source.base_offset;
+ tok->line_start = tok->buf_offset;
}
int
_PyTok_ReaderUnderflow(struct tok_state *tok)
{
- assert(tok->cur == tok->inp || (tok->buf != NULL &&
- tok->cur >= tok->buf && tok->cur < tok->inp));
+ assert(tok->cur >= tok->buf_offset && tok->cur <= tok->inp);
_PyTok_ReaderKind kind = tok->reader->kind;
int prepared = kind == _PYTOK_READER_PREPARED;
int streaming = reader_is_streaming(kind);
- int reset_buffer = !prepared && tok->start == NULL &&
+ int reset_buffer = !prepared && tok->start < 0 &&
_PyLexer_CurrentFTString(tok) == NULL;
_PyTok_Chunk chunk;
@@ -592,11 +588,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
if (streaming && reset_buffer) {
reset_streaming_buffer(tok);
}
- _PyLexer_BufferPointers pointers;
- if (!reset_buffer) {
- _PyLexer_SaveBufferPointers(
- tok, tok->source.bytes, &pointers);
- }
_PyTok_Off source_start = _PyTok_SourceAppendLine(
&tok->source, chunk.data, chunk.len,
chunk.implicit_newline);
@@ -607,37 +598,30 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
return 0;
}
if (reset_buffer) {
- tok->buf = tok->cur =
- tok->source.bytes + (source_start - tok->source.base_offset);
+ tok->cur = source_start;
tok->buf_offset = source_start;
- tok->line_start = tok->buf;
- tok->start = NULL;
- }
- else {
- _PyLexer_RestoreBufferPointers(
- tok, tok->source.bytes, &pointers);
+ tok->line_start = tok->buf_offset;
+ tok->start = -1;
}
- tok->inp = tok->source.bytes +
- (source_start - tok->source.base_offset) + scan_len;
+ tok->inp = source_start + scan_len;
}
if (tok->fp_interactive) {
tok->interactive_src_start = tok->source.bytes;
tok->interactive_src_end = tok->source.bytes + tok->source.len;
}
if (prepared) {
- if (tok->start == NULL && _PyLexer_CurrentFTString(tok) == NULL) {
- tok->buf = tok->cur;
+ if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) {
tok->buf_offset = tok->source.base_offset +
(chunk.data - tok->source.bytes);
}
- tok->inp = chunk.data + chunk.len;
+ tok->inp = _PyLexer_BufferOffset(tok, chunk.data) + chunk.len;
}
tok->implicit_newline = chunk.implicit_newline;
tok->lineno++;
if (kind == _PYTOK_READER_FILE &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
- !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
+ !_PyTokenizer_ensure_utf8(_PyLexer_BufferPointer(tok, tok->cur),
tok, tok->lineno)) {
_PyTok_ChunkClear(&chunk);
return 0;
}
@@ -663,9 +647,8 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
return tok;
}
if (reader_is_streaming(kind)) {
- tok->buf = tok->cur = tok->inp =
- (char *)_PyTok_SourceData(&tok->source);
- tok->line_start = tok->buf;
+ tok->cur = tok->inp = tok->source.base_offset;
+ tok->line_start = tok->buf_offset;
}
return tok;
}
@@ -683,8 +666,8 @@ tokenizer_from_string(const char *input, int utf8_only, int
exec_input,
_PyTokenizer_Free(tok);
return NULL;
}
- tok->buf = tok->cur = tok->inp = tok->str;
- tok->line_start = tok->str;
+ tok->cur = tok->inp = tok->source.base_offset;
+ tok->line_start = tok->source.base_offset;
return tok;
}
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index 7a2f46f73aff470..2f74ed8b1f4fab3 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -3,20 +3,7 @@
#include "Python.h"
-typedef Py_ssize_t _PyTok_Off;
-
-/* Spans use half-open logical byte offsets into decoded input. Their backing
- storage may retain only the current input window. */
-typedef struct {
- _PyTok_Off start;
- _PyTok_Off end;
-} _PyTok_Span;
-
-/* Lines are 1-based and byte columns are 0-based. */
-typedef struct {
- int lineno;
- int byte_col;
-} _PyTok_Loc;
+#include "types.h"
typedef enum {
_PYTOK_AFFINITY_LEFT,
@@ -77,18 +64,6 @@ PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
PyAPI_FUNC(int) _PyTok_SourceLocation(
const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
-static inline _PyTok_Span
-_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
-{
- return (_PyTok_Span){start, end};
-}
-
-static inline int
-_PyTok_SpanIsValid(_PyTok_Span span)
-{
- return span.start >= 0 && span.end >= span.start;
-}
-
static inline _PyTok_Off
_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
{
diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h
index d8c889115cfd73a..e9229d120871624 100644
--- a/Parser/tokenizer/tokenizer.h
+++ b/Parser/tokenizer/tokenizer.h
@@ -2,12 +2,98 @@
#define Py_TOKENIZER_H
#include "Python.h"
+#include "types.h"
-struct tok_state *_PyTokenizer_FromString(const char *, int, int);
-struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
-struct tok_state *_PyTokenizer_FromReadline(PyObject *, const char *);
-struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
- const char *, const char *);
+struct tok_state;
+
+/* Initialize before use. metadata owns a reference released by _PyToken_Free;
+ a consumer taking that reference must set metadata to NULL. */
+struct token {
+ int type;
+ int level;
+ int is_raw;
+ _PyTok_Span span;
+ _PyTok_Loc start_loc;
+ _PyTok_Loc end_loc;
+ PyObject *metadata;
+};
+
+typedef struct {
+ const char *text;
+ Py_ssize_t length;
+ const char *line;
+ Py_ssize_t line_length;
+ const char *end_line;
+ int implicit_newline;
+ int at_eof;
+} _PyToken_View;
+
+typedef struct {
+ int status;
+ _PyTok_Loc location;
+ _PyTok_Off cursor;
+ _PyTok_Span input_span;
+ _PyTok_Span line_span;
+ int level;
+ char delimiter;
+ _PyTok_Loc delimiter_loc;
+ int in_formatted_string;
+ int is_interactive;
+ int is_file;
+ PyObject *filename;
+ PyObject *module;
+ const char *encoding;
+} _PyTokenizer_Info;
+
+/* Get replaces the initialized token, releasing its previous metadata.
+ Errors are returned as ERRORTOKEN, with or without a Python exception. */
+void _PyTokenizer_Get(struct tok_state *, struct token *);
+void _PyTokenizer_Free(struct tok_state *);
+void _PyTokenizer_raise_init_error(PyObject *filename);
+void _PyToken_Init(struct token *);
+static inline void
+_PyToken_Free(struct token *token)
+{
+ Py_CLEAR(token->metadata);
+}
+
+/* Views and borrowed snapshot references remain valid until the tokenizer is
+ mutated or freed. Source spans may be discarded when reading more input. */
+_PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);
+/* An absent token span has a nonnull empty text view. */
+const char *_PyToken_TextView(
+ const struct tok_state *, const struct token *, Py_ssize_t *);
+/* Use the token from the most recent Get. text is NULL for an absent span;
+ line includes the token's complete physical line range. */
+void _PyToken_GetView(
+ const struct tok_state *tok, const struct token *token,
+ _PyToken_View *view);
+const char *_PyTokenizer_SpanView(
+ const struct tok_state *, _PyTok_Span, Py_ssize_t *);
+/* For retained input: 1-based lines clamp to the first or final line.
+ The view excludes the newline and need not be NUL-terminated. */
+const char *_PyTokenizer_LineView(
+ const struct tok_state *, Py_ssize_t, Py_ssize_t *);
+/* Return NUL-terminated retained input, or NULL without setting an exception
+ for streaming input or interactive input before its first line. */
+const char *_PyTokenizer_RetainedSource(const struct tok_state *);
+void _PyTokenizer_SetContext(
+ struct tok_state *tok, PyObject *filename, PyObject *module);
+void _PyTokenizer_SetOptions(
+ struct tok_state *tok, int extra_tokens, int type_comments);
+void _PyTokenizer_ImplyDedents(struct tok_state *);
+int _PyTokenizer_HasTrailingStatement(const struct tok_state *);
+int _PyTokenizer_IsInteractive(const struct tok_state *);
+void _PyTokenizer_StopInteractive(struct tok_state *);
+
+struct tok_state *_PyTokenizer_FromString(
+ const char *input, int exec_input, int preserve_crlf);
+struct tok_state *_PyTokenizer_FromUTF8(
+ const char *input, int exec_input, int preserve_crlf);
+struct tok_state *_PyTokenizer_FromReadline(
+ PyObject *readline, const char *encoding);
+struct tok_state *_PyTokenizer_FromFile(
+ FILE *fp, const char *encoding, const char *ps1, const char *ps2);
/* Return the declared encoding in PyMem-allocated storage, or NULL.
An exception is set on error. */
char *_PyTokenizer_FindEncodingFilename(int, PyObject *);
diff --git a/Parser/tokenizer/types.h b/Parser/tokenizer/types.h
new file mode 100644
index 000000000000000..c12b6de449c7008
--- /dev/null
+++ b/Parser/tokenizer/types.h
@@ -0,0 +1,33 @@
+#ifndef Py_TOKENIZER_TYPES_H
+#define Py_TOKENIZER_TYPES_H
+
+#include "Python.h"
+
+typedef Py_ssize_t _PyTok_Off;
+
+/* Spans use half-open logical byte offsets into decoded input. Their backing
+ storage may retain only the current input window. */
+typedef struct {
+ _PyTok_Off start;
+ _PyTok_Off end;
+} _PyTok_Span;
+
+/* Lines are 1-based and byte columns are 0-based. */
+typedef struct {
+ int lineno;
+ int byte_col;
+} _PyTok_Loc;
+
+static inline _PyTok_Span
+_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
+{
+ return (_PyTok_Span){start, end};
+}
+
+static inline int
+_PyTok_SpanIsValid(_PyTok_Span span)
+{
+ return span.start >= 0 && span.end >= span.start;
+}
+
+#endif
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index eb5c0b86a8fbe45..3802709732e29da 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -2,8 +2,6 @@
#include "errcode.h"
#include "internal/pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION
#include "internal/pycore_tuple.h" // _PyTuple_FromPair
-#include "../Parser/lexer/state.h"
-#include "../Parser/lexer/lexer.h"
#include "../Parser/tokenizer/tokenizer.h"
#include "../Parser/pegen.h" //
_PyPegen_byte_offset_to_character_offset()
@@ -34,6 +32,7 @@ typedef struct
{
PyObject_HEAD struct tok_state *tok;
int done;
+ int extra_tokens;
/* Needed to cache line for performance */
PyObject *last_line;
@@ -71,10 +70,10 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject
*readline,
Py_DECREF(filename);
return NULL;
}
- self->tok->filename = filename;
- if (extra_tokens) {
- self->tok->tok_extra_tokens = 1;
- }
+ _PyTokenizer_SetContext(self->tok, filename, NULL);
+ Py_DECREF(filename);
+ _PyTokenizer_SetOptions(self->tok, extra_tokens, 0);
+ self->extra_tokens = extra_tokens;
self->done = 0;
self->last_line = NULL;
@@ -96,14 +95,16 @@ _tokenizer_error(tokenizeriterobject *it)
const char *msg = NULL;
PyObject* errtype = PyExc_SyntaxError;
struct tok_state *tok = it->tok;
- switch (tok->done) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(tok);
+ switch (info.status) {
case E_TOKEN:
msg = "invalid token";
break;
case E_EOF:
PyErr_SetString(PyExc_SyntaxError, "unexpected EOF in multi-line
statement");
- PyErr_SyntaxLocationObject(tok->filename, tok->lineno,
- tok->inp - tok->buf < 0 ? 0 :
(int)(tok->inp - tok->buf));
+ PyErr_SyntaxLocationObject(
+ info.filename, info.location.lineno,
+ (int)Py_MAX(0, info.input_span.end - info.input_span.start));
return -1;
case E_DEDENT:
msg = "unindent does not match any outer indentation level";
@@ -139,21 +140,24 @@ _tokenizer_error(tokenizeriterobject *it)
PyObject* value = NULL;
int result = 0;
- Py_ssize_t size = tok->inp - tok->buf;
- assert(tok->buf[size-1] == '\n');
+ Py_ssize_t input_size;
+ const char *input = _PyTokenizer_SpanView(
+ tok, info.input_span, &input_size);
+ Py_ssize_t size = input_size;
+ assert(input[size-1] == '\n');
size -= 1; // Remove the newline character from the end of the line
- error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
+ error_line = PyUnicode_DecodeUTF8(input, size, "replace");
if (!error_line) {
result = -1;
goto exit;
}
- Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line,
tok->inp - tok->buf);
+ Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line,
input_size);
if (offset == -1) {
result = -1;
goto exit;
}
- tmp = Py_BuildValue("(OnnOOO)", tok->filename, tok->lineno, offset,
error_line, Py_None, Py_None);
+ tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno,
offset, error_line, Py_None, Py_None);
if (!tmp) {
result = -1;
goto exit;
@@ -182,12 +186,12 @@ _tokenizer_error(tokenizeriterobject *it)
}
static PyObject *
-_get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t
size,
- int *line_changed)
+_get_current_line(tokenizeriterobject *it, int current_lineno,
+ const char *line_start, Py_ssize_t size, int *line_changed)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
PyObject *line;
- if (it->tok->lineno != it->last_lineno) {
+ if (current_lineno != it->last_lineno) {
// Line has changed since last token, so we fetch the new line and
cache it
// in the iter object.
Py_XDECREF(it->last_line);
@@ -205,7 +209,7 @@ _get_current_line(tokenizeriterobject *it, const char
*line_start, Py_ssize_t si
static void
_get_col_offsets(tokenizeriterobject *it, const struct token *token,
const char *token_start, const char *line_start,
- PyObject *line, int line_changed,
+ const char *end_line_start, PyObject *line, int line_changed,
Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
@@ -225,8 +229,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct
token *token,
}
}
- if (token_end != NULL && token_end >= it->tok->line_start) {
- Py_ssize_t end_byte_offset = token_end - it->tok->line_start;
+ if (token_end != NULL && token_end >= end_line_start) {
+ Py_ssize_t end_byte_offset = token_end - end_line_start;
if (lineno == end_lineno) {
// Avoid rescanning the prefix of a very long line.
Py_ssize_t token_col_offset =
_PyPegen_byte_offset_to_character_offset_line(line, byte_offset,
end_byte_offset);
@@ -234,7 +238,7 @@ _get_col_offsets(tokenizeriterobject *it, const struct
token *token,
it->byte_col_offset_diff += token_end - token_start -
token_col_offset;
}
else {
- *end_col_offset =
_PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start,
end_byte_offset);
+ *end_col_offset =
_PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset);
it->byte_col_offset_diff += end_byte_offset - *end_col_offset;
}
}
@@ -253,7 +257,8 @@ tokenizeriter_next(PyObject *op)
struct token token;
_PyToken_Init(&token);
- int type = _PyTokenizer_Get(it->tok, &token);
+ _PyTokenizer_Get(it->tok, &token);
+ int type = token.type;
if (type == ERRORTOKEN) {
if(!PyErr_Occurred()) {
_tokenizer_error(it);
@@ -266,40 +271,38 @@ tokenizeriter_next(PyObject *op)
it->done = 1;
goto exit;
}
- const char *token_start = NULL;
+ _PyToken_View view;
+ _PyToken_GetView(it->tok, &token, &view);
+ const char *token_start = view.text;
PyObject *str;
if (token.span.start < 0) {
assert(token.span.start == -1 && token.span.end == -1);
str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
else {
- Py_ssize_t token_length;
- token_start = _PyToken_TextView(
- it->tok, &token, &token_length);
- str = PyUnicode_FromStringAndSize(token_start, token_length);
+ str = PyUnicode_FromStringAndSize(token_start, view.length);
}
if (str == NULL) {
goto exit;
}
int is_trailing_token = 0;
- if (type == ENDMARKER || (type == DEDENT && it->tok->done == E_EOF)) {
+ if (type == ENDMARKER || (type == DEDENT && view.at_eof)) {
is_trailing_token = 1;
}
- const char *line_start = ISSTRINGLIT(type)
- ? token_start - token.start_loc.byte_col : it->tok->line_start;
PyObject* line = NULL;
int line_changed = 1;
- if (it->tok->tok_extra_tokens && is_trailing_token) {
+ if (it->extra_tokens && is_trailing_token) {
line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
} else {
- Py_ssize_t size = it->tok->inp - line_start;
- if (size >= 1 && it->tok->implicit_newline) {
+ Py_ssize_t size = view.line_length;
+ if (size >= 1 && view.implicit_newline) {
size -= 1;
}
- line = _get_current_line(it, line_start, size, &line_changed);
+ line = _get_current_line(
+ it, token.end_loc.lineno, view.line, size, &line_changed);
}
if (line == NULL) {
Py_DECREF(str);
@@ -310,10 +313,10 @@ tokenizeriter_next(PyObject *op)
Py_ssize_t end_lineno = token.end_loc.lineno;
Py_ssize_t col_offset = -1;
Py_ssize_t end_col_offset = -1;
- _get_col_offsets(it, &token, token_start, line_start, line, line_changed,
- &col_offset, &end_col_offset);
+ _get_col_offsets(it, &token, token_start, view.line, view.end_line, line,
+ line_changed, &col_offset, &end_col_offset);
- if (it->tok->tok_extra_tokens) {
+ if (it->extra_tokens) {
if (is_trailing_token) {
lineno = end_lineno = lineno + 1;
col_offset = end_col_offset = 0;
@@ -325,7 +328,7 @@ tokenizeriter_next(PyObject *op)
}
else if (type == NEWLINE) {
Py_DECREF(str);
- if (!it->tok->implicit_newline) {
+ if (!view.implicit_newline) {
assert(token_start != NULL);
if (token_start[0] == '\r') {
str = PyUnicode_FromString("\r\n");
@@ -336,7 +339,7 @@ tokenizeriter_next(PyObject *op)
end_col_offset++;
}
else if (type == NL) {
- if (it->tok->implicit_newline) {
+ if (view.implicit_newline) {
Py_DECREF(str);
str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
diff --git a/Tools/peg_generator/pegen/build.py
b/Tools/peg_generator/pegen/build.py
index bfd8e43c6912e86..1dc33520e5387d9 100644
--- a/Tools/peg_generator/pegen/build.py
+++ b/Tools/peg_generator/pegen/build.py
@@ -128,8 +128,8 @@ def compile_c_extension(
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" /
"decoder.c"),
+ str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "api.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" /
"reader.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" /
"helpers.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
