https://github.com/python/cpython/commit/2a96282d9abbe0df49ee32d3f9d9f093705836bc
commit: 2a96282d9abbe0df49ee32d3f9d9f093705836bc
branch: main
author: Pablo Galindo Salgado <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-09-24T16:26:47Z
summary:
gh-153569: consolidate tokenizer input, layout, and diagnostics (#157055)
* gh-153569: move tokenizer input state and relocation into the reader
* gh-153569: borrow diagnostic lines through the source API
* gh-153569: group indentation and logical-line state
* gh-153569: remove unused tokenizer cursor and source lookup APIs
* gh-153569: report tokenizer diagnostics without rewinding the scanner
* gh-153569: use tokenizer views and remove obsolete API remnants
files:
A Parser/lexer/layout.c
D Parser/lexer/lexer.h
D Parser/tokenizer/cursor.c
D Parser/tokenizer/cursor.h
M Lib/test/test_capi/test_tokenizer.py
M Lib/test/test_codeop.py
M Lib/test/test_repl.py
M Lib/test/test_source_encoding.py
M Lib/test/test_tstring.py
M Makefile.pre.in
M Modules/_testinternalcapi/tokenizer.c
M PCbuild/_freeze_module.vcxproj
M PCbuild/_freeze_module.vcxproj.filters
M PCbuild/pythoncore.vcxproj
M PCbuild/pythoncore.vcxproj.filters
M Parser/lexer/lexer.c
M Parser/lexer/lexer_internal.h
M Parser/lexer/state.c
M Parser/lexer/state.h
M Parser/lexer/string.c
M Parser/pegen_errors.c
M Parser/tokenizer/api.c
M Parser/tokenizer/decoder.c
M Parser/tokenizer/helpers.c
M Parser/tokenizer/helpers.h
M Parser/tokenizer/reader.c
M Parser/tokenizer/reader.h
M Parser/tokenizer/reader_internal.h
M Parser/tokenizer/source.c
M Parser/tokenizer/source.h
M Parser/tokenizer/tokenizer.h
M Tools/peg_generator/pegen/build.py
diff --git a/Lib/test/test_capi/test_tokenizer.py
b/Lib/test/test_capi/test_tokenizer.py
index eb04f6c0136022..57d0a3c2f2e99e 100644
--- a/Lib/test/test_capi/test_tokenizer.py
+++ b/Lib/test/test_capi/test_tokenizer.py
@@ -12,9 +12,6 @@ def test_source(self):
def test_source_discard(self):
_testinternalcapi.test_tokenizer_source_discard()
- def test_cursor(self):
- _testinternalcapi.test_tokenizer_cursor()
-
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
index d57452602ce557..c75c0f8627bf08 100644
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -113,6 +113,17 @@ def test_valid(self, compiler):
av("def f():\n pass\n#foo\n")
av("@a.b.c\ndef f():\n pass\n")
+ @subTests('symbol', ('single', 'exec'))
+ @subTests('prefix', ('', 'f', 't'))
+ def test_incomplete_string_diagnostics(self, symbol, prefix):
+ opening = f' á = {prefix}"""first\n'
+ source = 'if True:\n' + opening + 'second'
+ with self.assertRaises(_IncompleteInputError) as cm:
+ Compile()(source, '<input>', symbol)
+ text = opening + 'second' + ('\n' if symbol == 'exec' else '')
+ self.assertEqual(cm.exception.args, (
+ 'incomplete input', ('<input>', 2, 9, text, 2, -1)))
+
@subTests('compiler', COMPILERS)
def test_incomplete(self, compiler):
ai = functools.partial(self.assertIncomplete, compiler=compiler)
diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
index ed6eb706c40d22..21e603c561d73c 100644
--- a/Lib/test/test_repl.py
+++ b/Lib/test/test_repl.py
@@ -185,9 +185,8 @@ def read_until(marker, start=0):
@cpython_only
def test_lexer_buffer_realloc_with_null_start(self):
- # gh-144759: NULL pointer arithmetic in the lexer when start and
- # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
- # and the lexer buffer is reallocated while parsing long input.
+ # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+ # while parsing long input.
long_value = "a" * 2000
user_input = dedent(f"""\
x = f'{{{long_value!r}}}'
diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index 862a20a058be75..ec98e609c4e98f 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -3,8 +3,8 @@
import unittest
from test import support
from test.support import script_helper
-from test.support.os_helper import TESTFN, unlink, rmtree
-from test.support.import_helper import unload
+from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
+from test.support.import_helper import import_module, unload
import importlib
import os
import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
def test_invalid_utf8_offset_after_non_ascii(self):
+ for name in ('é', 'éé', '𝒜'):
+ with self.subTest(name=name):
+ source = ('x = ' + name).encode() + b'\xff\n'
+ with self.assertRaises(SyntaxError) as caught:
+ compile(source, '<test>', 'exec')
+ error = caught.exception
+ self.assertEqual(
+ (error.lineno, error.offset, error.end_lineno,
error.end_offset),
+ (1, 5 + len(name), 1, 5 + len(name)),
+ )
+
+ @support.cpython_only
+ def test_invalid_utf8_file_offset_after_non_ascii(self):
+ _testcapi = import_module('_testcapi')
+ self.addCleanup(unlink, TESTFN_ASCII)
+ with open(TESTFN_ASCII, 'wb') as f:
+ f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
with self.assertRaises(SyntaxError) as caught:
- compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
+ _testcapi.run_file(
+ os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
error = caught.exception
self.assertEqual(
(error.lineno, error.offset, error.end_lineno, error.end_offset),
- (1, 6, 1, 6),
+ (2, 7, 2, 7),
)
def test_long_bom_conflict_message_is_not_truncated(self):
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 20a5083f60d11a..ea20688fdc9753 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -323,6 +323,8 @@ def test_nested_templates(self):
def test_syntax_errors(self):
for case, err in (
+ ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string
literal"),
+ ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string
literal"),
("t'", "unterminated t-string literal"),
("t'''", "unterminated triple-quoted t-string literal"),
("t''''", "unterminated triple-quoted t-string literal"),
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 71952252ac9f11..b29976ee041099 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -395,10 +395,10 @@ PEGEN_OBJS= \
TOKENIZER_OBJS= \
Parser/lexer/lexer.o \
+ Parser/lexer/layout.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
- Parser/tokenizer/cursor.o \
Parser/tokenizer/decoder.o \
Parser/tokenizer/api.o \
Parser/tokenizer/reader.o \
@@ -411,10 +411,8 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h
TOKENIZER_HEADERS= \
- Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
- Parser/tokenizer/cursor.h \
Parser/tokenizer/reader.h \
Parser/tokenizer/reader_internal.h \
Parser/tokenizer/source.h \
@@ -3471,7 +3469,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h
$(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h
$(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c
$(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h
$(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h
$(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h
$(srcdir)/Modules/_testlimitedcapi/parts.h
$(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
$(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h
$(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h
$(srcdir)/Modules/_testinternalcapi/test_targets.h
$(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
$(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h
$(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h
$(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h
$(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h
$(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h
$(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h
$(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
diff --git a/Modules/_testinternalcapi/tokenizer.c
b/Modules/_testinternalcapi/tokenizer.c
index df481cb832a436..1f89c12f223c7d 100644
--- a/Modules/_testinternalcapi/tokenizer.c
+++ b/Modules/_testinternalcapi/tokenizer.c
@@ -1,6 +1,6 @@
#include "parts.h"
-#include "../../Parser/tokenizer/cursor.h"
+#include "../../Parser/tokenizer/source.h"
static int
check(int condition, const char *message)
@@ -24,13 +24,14 @@ check_system_error(int failed, const char *message)
}
static int
-same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
+check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno,
+ const char *expected)
{
- return left->source == right->source &&
- left->pos == right->pos &&
- left->line_start == right->line_start &&
- left->line_end == right->line_end &&
- left->lineno == right->lineno;
+ Py_ssize_t len;
+ const char *line = _PyTok_SourceLineView(source, lineno, &len);
+ return check(len == (Py_ssize_t)strlen(expected) &&
+ memcmp(line, expected, len) == 0,
+ "wrong source line view");
}
static PyObject *
@@ -40,300 +41,54 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
_PyTok_SourceText source;
_PyTok_SourceInit(&source);
- _PyTok_Loc loc;
- _PyTok_Line line;
- if (check(_PyTok_SourceLocation(
- &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate empty source") < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 0,
- "wrong empty source location") < 0 ||
- check(_PyTok_SourceLine(&source, 1, &line) == 0,
- "cannot find empty source line") < 0 ||
- check(line.start == 0 && line.end == 0,
- "wrong empty source line") < 0 ||
- check_system_error(
- _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
- "accepted empty source line") < 0 ||
+ if (check_line_view(&source, 1, "") < 0) {
+ goto error;
+ }
+
+ if (check_system_error(
+ _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+ "accepted empty source line") < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
"accepted multiple source lines") < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
- "accepted missing implicit newline") < 0) {
- goto error;
- }
-
- if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0,
- "wrong first source offset") < 0 ||
+ "accepted missing implicit newline") < 0 ||
+ check(_PyTok_SourceAppendLine(
+ &source, "alpha\n", 6, 0) == 0,
+ "wrong first source offset") < 0 ||
check(_PyTok_SourceAppendLine(
&source, "\xce\xb2\n", 3, 1) == 6,
"wrong second source offset") < 0 ||
- check(_PyTok_SourceAppendLine(
- &source, "nul\0x\n", 6, 0) == 9,
- "wrong third source offset") < 0) {
- goto error;
- }
-
- int marker_line = 257;
- int final_line = 300;
- _PyTok_Off marker_start = -1;
- for (int lineno = 4; lineno <= final_line; lineno++) {
- const char *text = lineno == marker_line ? "marker\n" : "x\n";
- Py_ssize_t len = (Py_ssize_t)strlen(text);
- _PyTok_Off start = _PyTok_SourceAppendLine(
- &source, text, len, lineno == final_line);
- if (start < 0) {
- goto error;
- }
- if (lineno == marker_line) {
- marker_start = start;
- }
- }
-
- if (check(source.nlines == final_line, "wrong source line count") < 0 ||
- check(_PyTok_SourceLine(&source, marker_line, &line) == 0,
- "cannot find late source line") < 0 ||
- check(line.start == marker_start &&
- line.end == marker_start + 7,
- "wrong late source line") < 0 ||
- check(!line.implicit_newline && !line.contains_nul,
- "wrong late source flags") < 0 ||
- check(_PyTok_SourceLine(&source, 2, &line) == 0,
- "cannot find second source line") < 0 ||
- check(line.start == 6 && line.end == 9 &&
- line.implicit_newline && !line.contains_nul,
- "wrong second source line") < 0 ||
check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
_PyTok_SourceLineIsImplicit(&source, 2),
- "wrong early implicit newline flags") < 0 ||
- check(_PyTok_SourceLine(&source, 3, &line) == 0,
- "cannot find third source line") < 0 ||
- check(line.contains_nul, "missing null byte flag") < 0 ||
- check(_PyTok_SourceLine(&source, final_line, &line) == 0,
- "cannot find final source line") < 0 ||
- check(line.implicit_newline &&
- _PyTok_SourceLineIsImplicit(&source, final_line),
- "missing late implicit newline flag") < 0) {
+ "wrong implicit newline flags") < 0) {
goto error;
}
- Py_ssize_t view_len;
- const char *view = _PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(6, 8), &view_len);
- if (check(view != NULL && view_len == 2 &&
- memcmp(view, "\xce\xb2", 2) == 0,
- "wrong source span view") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start,
- _PYTOK_AFFINITY_LEFT, &loc) == 0,
- "cannot locate left line boundary") < 0 ||
- check(loc.lineno == marker_line - 1 && loc.byte_col == 2,
- "wrong left boundary location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate right line boundary") < 0 ||
- check(loc.lineno == marker_line && loc.byte_col == 0,
- "wrong right boundary location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start + 1,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate late source byte") < 0 ||
- check(loc.lineno == marker_line && loc.byte_col == 1,
- "wrong late source location") < 0) {
+ if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 ||
+ check_line_view(&source, 1, "alpha") < 0 ||
+ check_line_view(&source, 2, "\xce\xb2") < 0 ||
+ check_line_view(&source, 3, "") < 0 ||
+ check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) {
goto error;
}
- if (check(_PyTok_SourceLocation(
- &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0,
- "cannot locate left EOF") < 0 ||
- check(loc.lineno == final_line && loc.byte_col == 2,
- "wrong left EOF location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, source.len,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate right EOF") < 0 ||
- check(loc.lineno == final_line + 1 && loc.byte_col == 0,
- "wrong right EOF location") < 0 ||
- check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
- "cannot find virtual EOF line") < 0 ||
- check(line.start == source.len && line.end == source.len,
- "wrong virtual EOF line") < 0 ||
- check(!_PyTok_SourceLineIsImplicit(&source, 0) &&
- !_PyTok_SourceLineIsImplicit(
- &source, final_line + 1),
- "virtual or invalid line is implicit") < 0) {
- goto error;
- }
-
- view = _PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len);
- if (check_system_error(view == NULL, "accepted invalid source span") < 0 ||
- check_system_error(
- _PyTok_SourceLocation(
- &source, source.len + 1,
- _PYTOK_AFFINITY_RIGHT, &loc) < 0,
- "accepted invalid source offset") < 0 ||
- check_system_error(
- _PyTok_SourceLine(&source, final_line + 2, &line) < 0,
- "accepted invalid source line") < 0) {
+ if (check(source.len == 9 &&
+ memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0,
+ "wrong source contents") < 0) {
goto error;
}
_PyTok_SourceClear(&source);
- _PyTok_SourceInit(&source);
if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
- "appended after unterminated source line") < 0 ||
- check(_PyTok_SourceLocation(
- &source, source.len,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate unterminated EOF") < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 4,
- "wrong unterminated EOF location") < 0) {
- goto error;
- }
-
- _PyTok_SourceDiscard(&source);
- if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4,
- "wrong retained source offset") < 0 ||
- _PyTok_SourceLine(&source, 1, &line) < 0 ||
- check(line.start == 4 && line.end == 6,
- "wrong retained source line") < 0 ||
- _PyTok_SourceLocation(
- &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 0,
- "wrong retained source location") < 0) {
- goto error;
- }
- view = _PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(4, 5), &view_len);
- if (check(view != NULL && view_len == 1 && view[0] == 'a',
- "wrong retained source span") < 0 ||
- check_system_error(_PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL,
- "accepted discarded source span") < 0) {
- goto error;
- }
-
- _PyTok_SourceClear(&source);
- Py_RETURN_NONE;
-
-error:
- _PyTok_SourceClear(&source);
- return NULL;
-}
-
-static PyObject *
-test_tokenizer_cursor(PyObject *Py_UNUSED(module),
- PyObject *Py_UNUSED(args))
-{
- _PyTok_SourceText source;
- _PyTok_SourceInit(&source);
- if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
- _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) {
- goto error;
- }
-
- _PyTok_Cursor cursor;
- _PyTok_CursorInit(&cursor, &source);
- if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 ||
- check(cursor.lineno == 3 && cursor.pos == source.len,
- "wrong cursor at virtual EOF") < 0 ||
- _PyTok_CursorSetLine(&cursor, 1) < 0) {
- goto error;
- }
-
- char large[BUFSIZ + 1];
- memset(large, 'z', sizeof(large));
- large[sizeof(large) - 1] = '\n';
- if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) {
- goto error;
- }
-
- if (check(_PyTok_CursorPeek(&cursor, 0) == 'a',
- "wrong cursor peek after relocation") < 0 ||
- check(_PyTok_CursorPeek(&cursor, 1) == 'b',
- "wrong distant cursor peek") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'a',
- "wrong first cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'b',
- "wrong second cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == '\n',
- "wrong final cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == EOF,
- "cursor advanced past line") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
- "cannot seek cursor offset") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == '\n',
- "wrong cursor byte after seek") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
- "cannot seek line boundary") < 0 ||
- check(cursor.lineno == 2 && cursor.line_start == 3 &&
- _PyTok_CursorAdvance(&cursor) == 'c',
- "wrong cursor at line boundary") < 0 ||
- check(_PyTok_CursorSetLine(&cursor, 3) == 0,
- "cannot advance cursor to final line") < 0 ||
- check(cursor.line_start == 6 &&
- _PyTok_CursorAdvance(&cursor) == 'z',
- "wrong cursor byte on final line") < 0) {
- goto error;
- }
-
- _PyTok_Cursor saved = cursor;
- if (check_system_error(
- _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0,
- "accepted invalid cursor offset") < 0 ||
- check(same_cursor(&cursor, &saved),
- "invalid offset changed cursor") < 0 ||
- check_system_error(
- _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0,
- "accepted invalid cursor line") < 0 ||
- check(same_cursor(&cursor, &saved),
- "invalid line changed cursor") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, source.len) == 0,
- "cannot set cursor to EOF") < 0 ||
- check(cursor.lineno == 4 && cursor.pos == source.len,
- "wrong cursor at EOF") < 0) {
- goto error;
- }
-
-#if SIZEOF_VOID_P > 4
- char byte = 0;
- _PyTok_SourceText huge_source = {
- .bytes = &byte,
- .len = (_PyTok_Off)INT_MAX + 1,
- };
- _PyTok_Cursor huge_cursor = {
- .source = &huge_source,
- .pos = INT_MAX,
- .line_end = (_PyTok_Off)INT_MAX + 1,
- .lineno = 1,
- };
- if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
- huge_cursor.pos == INT_MAX,
- "cursor advanced past maximum column") < 0) {
- goto error;
- }
-#endif
-
- _PyTok_Off base = source.len;
- _PyTok_SourceDiscard(&source);
- if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
- _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) {
+ "appended after unterminated source line") < 0) {
goto error;
}
- _PyTok_CursorInit(&cursor, &source);
- if (_PyTok_CursorSetLine(&cursor, 1) < 0 ||
- check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b',
- "wrong retained cursor line") < 0 ||
- _PyTok_CursorSetLine(&cursor, 2) < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'c',
- "wrong retained cursor byte") < 0 ||
- _PyTok_CursorSetOffset(&cursor, base + 5) < 0 ||
- check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF,
- "wrong retained cursor EOF") < 0) {
+ if (check_line_view(&source, 1, "tail") < 0 ||
+ check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) {
goto error;
}
@@ -410,7 +165,6 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
static PyMethodDef test_methods[] = {
{"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
- {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
{"test_tokenizer_source_discard", test_tokenizer_source_discard,
METH_NOARGS},
{NULL},
};
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 69833f132b5e4d..36a1d768486a3f 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -183,6 +183,7 @@
<ClCompile Include="..\Parser\token.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
+ <ClCompile Include="..\Parser\lexer\layout.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
diff --git a/PCbuild/_freeze_module.vcxproj.filters
b/PCbuild/_freeze_module.vcxproj.filters
index 207552113c3dd2..b358e09950a443 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -463,6 +463,9 @@
<ClCompile Include="..\Parser\lexer\lexer.c">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\Parser\lexer\layout.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="..\Parser\lexer\number.c">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index e4766fd7e5f764..c2d8f64a72b119 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -422,9 +422,7 @@
<ClInclude Include="..\Objects\stringlib\split.h" />
<ClInclude Include="..\Objects\unicodetype_db.h" />
<ClInclude Include="..\Parser\lexer\state.h" />
- <ClInclude Include="..\Parser\lexer\lexer.h" />
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
- <ClInclude Include="..\Parser\tokenizer\cursor.h" />
<ClInclude Include="..\Parser\tokenizer\reader.h" />
<ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
<ClInclude Include="..\Parser\tokenizer\types.h" />
@@ -595,9 +593,9 @@
<ClCompile Include="..\Parser\myreadline.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
+ <ClCompile Include="..\Parser\lexer\layout.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
- <ClCompile Include="..\Parser\tokenizer\cursor.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
<ClCompile Include="..\Parser\tokenizer\api.c" />
diff --git a/PCbuild/pythoncore.vcxproj.filters
b/PCbuild/pythoncore.vcxproj.filters
index 993c7ed24d56f4..2536fc971700de 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -321,18 +321,12 @@
<ClInclude Include="..\Objects\unicodetype_db.h">
<Filter>Objects</Filter>
</ClInclude>
- <ClInclude Include="..\Parser\lexer\lexer.h">
- <Filter>Parser</Filter>
- </ClInclude>
<ClInclude Include="..\Parser\lexer\state.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\lexer\lexer_internal.h">
<Filter>Parser</Filter>
</ClInclude>
- <ClInclude Include="..\Parser\tokenizer\cursor.h">
- <Filter>Parser</Filter>
- </ClInclude>
<ClInclude Include="..\Parser\tokenizer\reader.h">
<Filter>Parser</Filter>
</ClInclude>
@@ -1355,6 +1349,9 @@
<ClCompile Include="..\Parser\lexer\lexer.c">
<Filter>Parser</Filter>
</ClCompile>
+ <ClCompile Include="..\Parser\lexer\layout.c">
+ <Filter>Parser</Filter>
+ </ClCompile>
<ClCompile Include="..\Parser\lexer\number.c">
<Filter>Parser</Filter>
</ClCompile>
@@ -1364,9 +1361,6 @@
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Parser</Filter>
</ClCompile>
- <ClCompile Include="..\Parser\tokenizer\cursor.c">
- <Filter>Parser</Filter>
- </ClCompile>
<ClCompile Include="..\Parser\tokenizer\source.c">
<Filter>Parser</Filter>
</ClCompile>
diff --git a/Parser/lexer/layout.c b/Parser/lexer/layout.c
new file mode 100644
index 00000000000000..6091f9adeac958
--- /dev/null
+++ b/Parser/lexer/layout.c
@@ -0,0 +1,195 @@
+#include "Python.h"
+#include "errcode.h"
+#include "pycore_token.h"
+
+#include "lexer_internal.h"
+#include "../tokenizer/helpers.h"
+#include "../tokenizer/reader.h"
+
+#define TABSIZE 8
+#define ALTTABSIZE 1
+
+int
+_PyLexer_ContinueLine(struct tok_state *tok)
+{
+ int c = tok_nextc(tok);
+ if (c == '\r') {
+ c = tok_nextc(tok);
+ }
+ if (c != '\n') {
+ tok->done = E_LINECONT;
+ return -1;
+ }
+ c = tok_nextc(tok);
+ if (c == EOF) {
+ tok->done = E_EOF;
+ tok->cur = tok->inp;
+ return -1;
+ } else {
+ tok_backup(tok, c);
+ }
+ return c;
+}
+
+
+static int
+update_indentation(struct tok_state *tok, int col, int altcol)
+{
+ lexer_layout_state *layout = &tok->layout;
+ if (col == layout->stack[layout->depth].column) {
+ if (altcol != layout->stack[layout->depth].alternate_column) {
+ _PyTokenizer_indenterror(tok);
+ return -1;
+ }
+ }
+ else if (col > layout->stack[layout->depth].column) {
+ if (layout->depth + 1 >= MAXINDENT) {
+ tok->done = E_TOODEEP;
+ tok->cur = tok->inp;
+ return -1;
+ }
+ if (altcol <= layout->stack[layout->depth].alternate_column) {
+ _PyTokenizer_indenterror(tok);
+ return -1;
+ }
+ layout->pending++;
+ layout->stack[++layout->depth] = (indentation_level){col, altcol};
+ }
+ else {
+ while (layout->depth > 0 &&
+ col < layout->stack[layout->depth].column) {
+ layout->pending--;
+ layout->depth--;
+ }
+ if (col != layout->stack[layout->depth].column) {
+ tok->done = E_DEDENT;
+ tok->cur = tok->inp;
+ return -1;
+ }
+ if (altcol != layout->stack[layout->depth].alternate_column) {
+ _PyTokenizer_indenterror(tok);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+int
+_PyLexer_BeginLine(struct tok_state *tok)
+{
+ assert(tok->layout.at_bol);
+ int c;
+ int blankline = 0;
+ int col = 0;
+ int altcol = 0;
+ tok->layout.at_bol = 0;
+ int cont_line_col = 0;
+ for (;;) {
+ c = tok_nextc(tok);
+ if (c == ' ') {
+ col++, altcol++;
+ }
+ else if (c == '\t') {
+ col = (col / TABSIZE + 1) * TABSIZE;
+ altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
+ }
+ else if (c == '\014') {/* Control-L (formfeed) */
+ col = altcol = 0; /* For Emacs users */
+ }
+ else if (c == '\\') {
+ // Indentation cannot be split over multiple physical lines
+ // using backslashes. This means that if we found a backslash
+ // preceded by whitespace, **the first one we find** determines
+ // the level of indentation of whatever comes next.
+ cont_line_col = cont_line_col ? cont_line_col : col;
+ if ((c = _PyLexer_ContinueLine(tok)) == -1) {
+ return -1;
+ }
+ }
+ else if (c == EOF && PyErr_Occurred()) {
+ return -1;
+ }
+ else {
+ break;
+ }
+ }
+ tok_backup(tok, c);
+ if (c == '#' || c == '\n' || c == '\r') {
+ int interactive = _PyTok_ReaderIsInteractive(tok);
+ /* Lines with only whitespace and/or comments
+ shouldn't affect the indentation and are
+ not passed to the parser as NEWLINE tokens,
+ except *totally* empty lines in interactive
+ mode, which signal the end of a command group. */
+ if (col == 0 && c == '\n' && interactive) {
+ blankline = 0; /* Let it through */
+ }
+ else if (interactive && tok->lineno == 1) {
+ /* In interactive mode, if the first line contains
+ only spaces and/or a comment, let it through. */
+ blankline = 0;
+ col = altcol = 0;
+ }
+ else {
+ blankline = 1; /* Ignore completely */
+ }
+ }
+ if (!blankline && tok->level == 0) {
+ col = cont_line_col ? cont_line_col : col;
+ altcol = cont_line_col ? cont_line_col : altcol;
+ if (update_indentation(tok, col, altcol) < 0) {
+ return -1;
+ }
+ }
+ return blankline;
+}
+
+int
+_PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
+{
+ assert(tok->layout.pending != 0);
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
+ if (tok->layout.pending < 0) {
+ if (tok->tok_extra_tokens) {
+ p_start = tok->cur;
+ p_end = tok->cur;
+ }
+ tok->layout.pending++;
+ return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
+ }
+ else {
+ if (tok->tok_extra_tokens) {
+ p_start = tok->buf_offset;
+ p_end = tok->cur;
+ }
+ tok->layout.pending--;
+ return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
+ }
+}
+
+int
+_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
+{
+ tok->layout.at_bol = 1;
+ if (blankline || tok->level > 0) {
+ if (!tok->tok_extra_tokens) {
+ return 0;
+ }
+ }
+ else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) {
+ return _PyLexer_token_setup(tok, token, NEWLINE,
+ tok->start, tok->cur - 1);
+ }
+ tok->layout.comment_newline = 0;
+ return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
+}
+
+void
+_PyLexer_ImplyDedents(struct tok_state *tok)
+{
+ if (tok->layout.depth != 0) {
+ tok->layout.pending = -tok->layout.depth;
+ tok->layout.depth = 0;
+ }
+}
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index 110a225750f055..f27fee8d61b908 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -7,10 +7,6 @@
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"
-#define TABSIZE 8
-#define ALTTABSIZE 1
-
-
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type,
p_start, p_end)
/* Spaces in this constant are treated as "zero or more spaces or tabs" when
@@ -93,6 +89,7 @@ verify_identifier(struct tok_state *tok)
assert(PyUnicode_GET_LENGTH(s) > 0);
if (invalid < PyUnicode_GET_LENGTH(s)) {
Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
+ _PyTok_Off error_cursor = tok->cur;
if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
/* Determine the offset in UTF-8 encoded input */
Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
@@ -103,14 +100,20 @@ verify_identifier(struct tok_state *tok)
tok->done = E_ERROR;
return 0;
}
- tok->cur = tok->start + PyBytes_GET_SIZE(s);
+ error_cursor = tok->start + PyBytes_GET_SIZE(s);
}
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
- _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)",
ch, ch);
+ _PyTokenizer_syntaxerror_at(
+ tok, _PyLexer_BufferPointer(tok, tok->line_start),
+ error_cursor - tok->line_start, tok->lineno, -1, -1,
+ "invalid character '%c' (U+%04X)", ch, ch);
}
else {
- _PyTokenizer_syntaxerror(tok, "invalid non-printable character
U+%04X", ch);
+ _PyTokenizer_syntaxerror_at(
+ tok, _PyLexer_BufferPointer(tok, tok->line_start),
+ error_cursor - tok->line_start, tok->lineno, -1, -1,
+ "invalid non-printable character U+%04X", ch);
}
return 0;
}
@@ -118,31 +121,6 @@ verify_identifier(struct tok_state *tok)
return 1;
}
-
-
-static inline int
-tok_continuation_line(struct tok_state *tok) {
- int c = tok_nextc(tok);
- if (c == '\r') {
- c = tok_nextc(tok);
- }
- if (c != '\n') {
- tok->done = E_LINECONT;
- return -1;
- }
- c = tok_nextc(tok);
- if (c == EOF) {
- tok->done = E_EOF;
- tok->cur = tok->inp;
- return -1;
- } else {
- tok_backup(tok, c);
- }
- return c;
-}
-
-
-
int
_PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct
token *token)
{
@@ -160,102 +138,10 @@ _PyLexer_get_normal(struct tok_state *tok,
ftstring_state *current, struct token
blankline = 0;
- /* Get indentation level */
- if (tok->atbol) {
- int col = 0;
- int altcol = 0;
- tok->atbol = 0;
- int cont_line_col = 0;
- for (;;) {
- c = tok_nextc(tok);
- if (c == ' ') {
- col++, altcol++;
- }
- else if (c == '\t') {
- col = (col / TABSIZE + 1) * TABSIZE;
- altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
- }
- else if (c == '\014') {/* Control-L (formfeed) */
- col = altcol = 0; /* For Emacs users */
- }
- else if (c == '\\') {
- // Indentation cannot be split over multiple physical lines
- // using backslashes. This means that if we found a backslash
- // preceded by whitespace, **the first one we find** determines
- // the level of indentation of whatever comes next.
- cont_line_col = cont_line_col ? cont_line_col : col;
- if ((c = tok_continuation_line(tok)) == -1) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- else if (c == EOF && PyErr_Occurred()) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- else {
- break;
- }
- }
- tok_backup(tok, c);
- if (c == '#' || c == '\n' || c == '\r') {
- /* Lines with only whitespace and/or comments
- shouldn't affect the indentation and are
- not passed to the parser as NEWLINE tokens,
- except *totally* empty lines in interactive
- mode, which signal the end of a command group. */
- if (col == 0 && c == '\n' && tok->prompt != NULL) {
- blankline = 0; /* Let it through */
- }
- else if (tok->prompt != NULL && tok->lineno == 1) {
- /* In interactive mode, if the first line contains
- only spaces and/or a comment, let it through. */
- blankline = 0;
- col = altcol = 0;
- }
- else {
- blankline = 1; /* Ignore completely */
- }
- /* We can't jump back right here since we still
- may need to skip to the end of a comment */
- }
- if (!blankline && tok->level == 0) {
- col = cont_line_col ? cont_line_col : col;
- altcol = cont_line_col ? cont_line_col : altcol;
- if (col == tok->indstack[tok->indent]) {
- /* No change */
- if (altcol != tok->altindstack[tok->indent]) {
- return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
- }
- }
- else if (col > tok->indstack[tok->indent]) {
- /* Indent -- always one */
- if (tok->indent+1 >= MAXINDENT) {
- tok->done = E_TOODEEP;
- tok->cur = tok->inp;
- return MAKE_TOKEN(ERRORTOKEN);
- }
- if (altcol <= tok->altindstack[tok->indent]) {
- return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
- }
- tok->pendin++;
- tok->indstack[++tok->indent] = col;
- tok->altindstack[tok->indent] = altcol;
- }
- else /* col < tok->indstack[tok->indent] */ {
- /* Dedent -- any number, must be consistent */
- while (tok->indent > 0 &&
- col < tok->indstack[tok->indent]) {
- tok->pendin--;
- tok->indent--;
- }
- if (col != tok->indstack[tok->indent]) {
- tok->done = E_DEDENT;
- tok->cur = tok->inp;
- return MAKE_TOKEN(ERRORTOKEN);
- }
- if (altcol != tok->altindstack[tok->indent]) {
- return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
- }
- }
+ if (tok->layout.at_bol) {
+ blankline = _PyLexer_BeginLine(tok);
+ if (blankline < 0) {
+ return MAKE_TOKEN(ERRORTOKEN);
}
}
@@ -263,24 +149,8 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
tok->start_loc = (_PyTok_Loc){
tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1};
- /* Return pending indents/dedents */
- if (tok->pendin != 0) {
- if (tok->pendin < 0) {
- if (tok->tok_extra_tokens) {
- p_start = tok->cur;
- p_end = tok->cur;
- }
- tok->pendin++;
- return MAKE_TOKEN(DEDENT);
- }
- else {
- if (tok->tok_extra_tokens) {
- p_start = tok->buf_offset;
- p_end = tok->cur;
- }
- tok->pendin--;
- return MAKE_TOKEN(INDENT);
- }
+ if (tok->layout.pending != 0) {
+ return _PyLexer_IndentationToken(tok, token);
}
/* Peek ahead at the next character */
@@ -373,7 +243,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
/* If this type ignore is the only thing on the line,
consume the newline also. */
if (blankline) {
tok_nextc(tok);
- tok->atbol = 1;
+ tok->layout.at_bol = 1;
}
} else {
p_start = _PyLexer_BufferOffset(tok, type_start);
@@ -389,7 +259,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
tok_backup(tok, c); /* don't eat the newline or EOF */
p_start = _PyLexer_BufferOffset(tok, p);
p_end = tok->cur;
- tok->comment_newline = blankline;
+ tok->layout.comment_newline = blankline;
return MAKE_TOKEN(COMMENT);
}
}
@@ -470,29 +340,12 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
c = tok_nextc(tok);
}
- /* Newline */
if (c == '\n') {
- tok->atbol = 1;
- if (blankline || tok->level > 0) {
- if (tok->tok_extra_tokens) {
- if (tok->comment_newline) {
- tok->comment_newline = 0;
- }
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(NL);
- }
+ int type = _PyLexer_Newline(tok, token, blankline);
+ if (type == 0) {
goto nextline;
}
- if (tok->comment_newline && tok->tok_extra_tokens) {
- tok->comment_newline = 0;
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(NL);
- }
- p_start = tok->start;
- p_end = tok->cur - 1; /* Leave '\n' out of the string */
- return MAKE_TOKEN(NEWLINE);
+ return type;
}
/* Period or number starting with period? */
@@ -533,7 +386,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state
*current, struct token
/* Line continuation */
if (c == '\\') {
- if ((c = tok_continuation_line(tok)) == -1) {
+ if ((c = _PyLexer_ContinueLine(tok)) == -1) {
return MAKE_TOKEN(ERRORTOKEN);
}
goto again; /* Read next line */
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
deleted file mode 100644
index 7302198cc91010..00000000000000
--- a/Parser/lexer/lexer.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef _PY_LEXER_LEXER_H_
-#define _PY_LEXER_LEXER_H_
-
-#include "state.h"
-
-#endif
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index 210d182d42dd9f..f0d9576d95f8ff 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -2,7 +2,7 @@
#define _PY_LEXER_INTERNAL_H_
#include "errcode.h"
-#include "lexer.h"
+#include "state.h"
#define is_potential_identifier_start(c) (\
(c >= 'a' && c <= 'z')\
@@ -50,6 +50,12 @@ tok_nextc(struct tok_state *tok)
tok->source.bytes[tok->cur++ - tok->source.base_offset]);
}
+/* Return -1 on error, otherwise whether the line is blank. */
+int _PyLexer_BeginLine(struct tok_state *);
+int _PyLexer_ContinueLine(struct tok_state *);
+int _PyLexer_IndentationToken(struct tok_state *, struct token *);
+/* Return zero when the newline is suppressed, otherwise its token type. */
+int _PyLexer_Newline(struct tok_state *, struct token *, int);
void _PyLexer_backup(struct tok_state *, int);
int _PyLexer_record_ftstring_comment(
struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index a6617c33480c85..75ff26f16d47ba 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -1,5 +1,4 @@
#include "Python.h"
-#include "pycore_pystate.h"
#include "pycore_token.h"
#include "errcode.h"
@@ -7,55 +6,6 @@
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"
-/* Create and initialize a new tok_state structure */
-struct tok_state *
-_PyTokenizer_tok_new(void)
-{
- struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
- 1,
- sizeof(struct tok_state));
- if (tok == NULL) {
- PyErr_NoMemory();
- return NULL;
- }
-
- tok->cur = tok->inp = 0;
- tok->line_start = -1;
- tok->fp_interactive = 0;
- tok->interactive_src_start = NULL;
- tok->interactive_src_end = NULL;
- tok->start = -1;
- tok->done = E_OK;
- tok->fp = NULL;
- tok->indent = 0;
- tok->indstack[0] = 0;
- tok->atbol = 1;
- tok->pendin = 0;
- tok->prompt = NULL;
- tok->lineno = 0;
- tok->start_loc = (_PyTok_Loc){-1, -1};
- tok->level = 0;
- tok->altindstack[0] = 0;
- tok->encoding = NULL;
- tok->filename = NULL;
- tok->module = NULL;
- tok->type_comments = 0;
- tok->interactive_underflow = IUNDERFLOW_NORMAL;
- tok->str = NULL;
- tok->report_warnings = 1;
- tok->tok_extra_tokens = 0;
- tok->comment_newline = 0;
- tok->implicit_newline = 0;
- _PyTok_SourceInit(&tok->source);
- tok->reader = NULL;
- tok->ftstring_stack = tok->ftstring_stack_inline;
- tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY;
-#ifdef Py_DEBUG
- tok->debug = _Py_GetConfig()->parser_debug;
-#endif
- return tok;
-}
-
ftstring_state *
_PyLexer_PushFTString(struct tok_state *tok)
{
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 0f9ddb6d45e961..dff67c2ba83dae 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -10,15 +10,6 @@
#define MAXFTSTRINGLEVEL 150
#define FTSTRING_STACK_INLINE_CAPACITY 1
-enum interactive_underflow_t {
- /* Normal mode of operation: return a new token when asked in interactive
mode */
- IUNDERFLOW_NORMAL,
- /* Forcefully return ENDMARKER when asked for a new token in interactive
mode. This
- * can be used to prevent the tokenizer to prompt the user for new tokens
*/
- IUNDERFLOW_STOP,
-};
-
-
typedef enum {
FTSTRING_MODE_MIDDLE,
FTSTRING_MODE_EXPRESSION,
@@ -66,6 +57,19 @@ _PyLexer_IsRawString(ftstring_kind kind)
return kind == RAW_FSTRING || kind == RAW_TSTRING;
}
+typedef struct {
+ int column;
+ int alternate_column;
+} indentation_level;
+
+typedef struct {
+ int depth;
+ int pending;
+ int at_bol;
+ int comment_newline;
+ indentation_level stack[MAXINDENT];
+} lexer_layout_state;
+
/* Tokenizer state */
struct tok_state {
_PyTok_Off buf_offset;
@@ -74,19 +78,13 @@ struct tok_state {
_PyTok_Off start;
_PyTok_Off line_start;
_PyTok_SourceText source;
- int fp_interactive; /* If the file descriptor is interactive */
- char *interactive_src_start; /* The start of the source parsed so far in
interactive mode */
- char *interactive_src_end; /* The end of the source parsed so far in
interactive mode */
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
- int indent; /* Current indentation index */
- int indstack[MAXINDENT]; /* Stack of indents */
- int atbol; /* Nonzero if at begin of new line */
- int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
- const char *prompt; /* For interactive prompting */
+ lexer_layout_state layout;
int lineno; /* Current line number */
_PyTok_Loc start_loc;
+ _PyTokenizer_Diagnostic diagnostic;
int level; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
char parenstack[MAXLEVEL];
@@ -94,25 +92,18 @@ struct tok_state {
int parencolstack[MAXLEVEL];
PyObject *filename;
PyObject *module;
- /* Stuff for checking on different tab sizes */
- int altindstack[MAXINDENT]; /* Stack of alternate indents */
/* Stuff for PEP 0263 */
char *encoding; /* Source encoding. */
- char* str; /* Source string being tokenized (if tokenizing from a
string)*/
struct _PyTok_Reader *reader;
int type_comments; /* Whether to look for type comments */
- /* How to proceed when asked for a new token in interactive mode */
- enum interactive_underflow_t interactive_underflow;
- int report_warnings;
ftstring_state *ftstring_stack;
ftstring_state ftstring_stack_inline[FTSTRING_STACK_INLINE_CAPACITY];
int ftstring_depth;
int ftstring_capacity;
int tok_extra_tokens;
- int comment_newline;
int implicit_newline;
#ifdef Py_DEBUG
int debug;
@@ -182,7 +173,8 @@ _PyLexer_ByteColumn(const struct tok_state *tok)
int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type,
_PyTok_Off start, _PyTok_Off end);
-struct tok_state *_PyTokenizer_tok_new(void);
+void _PyLexer_ImplyDedents(struct tok_state *);
+
void _PyTokenizer_Free(struct tok_state *);
ftstring_state *_PyLexer_PushFTString(struct tok_state *);
void _PyLexer_PopFTString(struct tok_state *);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index 1fb684e337de97..a48cedf270362a 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -7,13 +7,18 @@
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type,
p_start, p_end)
-static void
-rewind_to_string_start(struct tok_state *tok, _PyTok_Off start,
- _PyTok_Loc location)
+static int
+string_error_token(struct tok_state *tok, struct token *token,
+ _PyTok_Off start, _PyTok_Loc location)
{
- tok->cur = start + 1;
- tok->line_start = start - location.byte_col;
- tok->lineno = location.lineno;
+ tok->diagnostic = (_PyTokenizer_Diagnostic){
+ .location = {location.lineno, location.byte_col + 1},
+ .text_span = _PyTok_SpanFromBounds(start - location.byte_col,
tok->inp),
+ };
+ int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1);
+ token->start_loc = location;
+ token->end_loc = (_PyTok_Loc){location.lineno, -1};
+ return type;
}
int
@@ -351,7 +356,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token
*token, int c)
}
if (c == EOF || (quote_size == 1 && c == '\n')) {
int end_lineno = tok->lineno;
- rewind_to_string_start(tok, tok->start, tok->start_loc);
+ _PyTok_Loc location = tok->start_loc;
+ const char *line = _PyLexer_BufferPointer(tok, tok->start) -
location.byte_col;
+ Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
const ftstring_state *state = _PyLexer_CurrentFTString(tok);
if (state != NULL) {
@@ -364,41 +371,49 @@ _PyLexer_scan_string(struct tok_state *tok, struct token
*token, int c)
assert(tok->parenstack[level] == '{');
int lineno = tok->parenlinenostack[level];
if (lineno != tok->lineno) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
"%c-string: expecting '}' to close '{' on line %d",
- _PyLexer_StringPrefix(state->kind), lineno));
+ _PyLexer_StringPrefix(state->kind), lineno);
+ }
+ else {
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "%c-string: expecting '}'",
+ _PyLexer_StringPrefix(state->kind));
}
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "%c-string: expecting '}'",
- _PyLexer_StringPrefix(state->kind)));
+ return string_error_token(tok, token, tok->start,
location);
}
}
if (quote_size == 3) {
- _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted
string literal"
- " (detected at line %d)", end_lineno);
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated triple-quoted string literal"
+ " (detected at line %d)", end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
- return MAKE_TOKEN(ERRORTOKEN);
+ return string_error_token(tok, token, tok->start, location);
}
else {
if (has_escaped_quote) {
- _PyTokenizer_syntaxerror(
- tok,
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
"unterminated string literal (detected at line %d); "
"perhaps you escaped the end quote?",
end_lineno
);
} else {
- _PyTokenizer_syntaxerror(
- tok, "unterminated string literal (detected at line
%d)", end_lineno
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated string literal (detected at line %d)",
end_lineno
);
}
if (c != '\n') {
tok->done = E_EOLS;
}
- return MAKE_TOKEN(ERRORTOKEN);
+ return string_error_token(tok, token, tok->start, location);
}
}
if (c == quote) {
@@ -462,25 +477,29 @@ _PyLexer_get_ftstring(struct tok_state *tok,
ftstring_state *current, struct tok
}
int end_lineno = tok->lineno;
- rewind_to_string_start(tok,
- current->start,
- current->start_loc);
+ _PyTok_Loc location = current->start_loc;
+ const char *line = _PyLexer_BufferPointer(tok, current->start) -
location.byte_col;
+ Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
if (quote_size == 3) {
- _PyTokenizer_syntaxerror(tok,
- "unterminated triple-quoted %c-string
literal"
- " (detected at line %d)",
- _PyLexer_StringPrefix(current->kind),
end_lineno);
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated triple-quoted %c-string literal"
+ " (detected at line %d)",
+ _PyLexer_StringPrefix(current->kind), end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
- return MAKE_TOKEN(ERRORTOKEN);
+ return string_error_token(tok, token,
+ current->start, location);
}
else {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "unterminated %c-string literal (detected
at"
- " line %d)",
- _PyLexer_StringPrefix(current->kind),
end_lineno));
+ _PyTokenizer_syntaxerror_at(
+ tok, line, cursor_offset, location.lineno, -1, -1,
+ "unterminated %c-string literal (detected at line %d)",
+ _PyLexer_StringPrefix(current->kind), end_lineno);
+ return string_error_token(tok, token,
+ current->start, location);
}
}
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index 7841f01b612915..74ab56c912ae1a 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -206,7 +206,9 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int
use_mark, const char *err
Py_ssize_t end_col_offset = -1;
if (t->col_offset == -1) {
_PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
- if (info.cursor == info.input_span.start) {
+ if (info.diagnostic.location.lineno != 0) {
+ col_offset = info.diagnostic.location.byte_col;
+ } else if (info.cursor == info.input_span.start) {
col_offset = 0;
} else {
col_offset = Py_SAFE_DOWNCAST(
@@ -256,8 +258,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject
*errtype,
PyObject *tmp = NULL;
p->error_indicator = 1;
_PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
- _PyTok_Loc location = info.location;
- _PyTok_Span text_span = info.line_span;
+ _PyTok_Loc location = info.diagnostic.location.lineno != 0
+ ? info.diagnostic.location : info.location;
+ _PyTok_Span text_span = info.diagnostic.location.lineno != 0
+ ? info.diagnostic.text_span : info.line_span;
if (end_lineno == CURRENT_POS) {
end_lineno = location.lineno;
diff --git a/Parser/tokenizer/api.c b/Parser/tokenizer/api.c
index 67e129d52441c7..3f5efa8dc8f26e 100644
--- a/Parser/tokenizer/api.c
+++ b/Parser/tokenizer/api.c
@@ -4,7 +4,6 @@
#include "tokenizer.h"
#include "reader.h"
-#include "reader_internal.h"
#include "../lexer/state.h"
_PyTokenizer_Info
@@ -12,6 +11,7 @@ _PyTokenizer_GetInfo(const struct tok_state *tok)
{
_PyTokenizer_Info info = {
.status = tok->done,
+ .diagnostic = tok->diagnostic,
.location = {tok->lineno, tok->line_start < 0
? -1 : (int)(tok->cur - tok->line_start)},
.cursor = tok->cur,
@@ -20,7 +20,7 @@ _PyTokenizer_GetInfo(const struct tok_state *tok)
.level = tok->level,
.delimiter_loc = {-1, -1},
.in_formatted_string = tok->ftstring_depth != 0,
- .is_interactive = tok->reader->kind == _PYTOK_READER_INTERACTIVE,
+ .is_interactive = _PyTok_ReaderIsInteractive(tok),
.is_file = tok->fp != NULL && tok->fp != stdin,
.filename = tok->filename,
.module = tok->module,
@@ -81,32 +81,7 @@ const char *
_PyTokenizer_LineView(const struct tok_state *tok, Py_ssize_t lineno,
Py_ssize_t *length)
{
- const char *line = _PyTokenizer_RetainedSource(tok);
- if (line == NULL) {
- line = _PyLexer_BufferPointer(tok, tok->buf_offset);
- }
- for (Py_ssize_t i = 1; i < lineno; i++) {
- const char *next = strchr(line, '\n');
- if (next == NULL) {
- break;
- }
- line = next + 1;
- }
- const char *end = strchr(line, '\n');
- *length = end != NULL ? end - line : (Py_ssize_t)strlen(line);
- return line;
-}
-
-const char *
-_PyTokenizer_RetainedSource(const struct tok_state *tok)
-{
- if (tok->reader->kind == _PYTOK_READER_PREPARED) {
- return _PyTok_SourceData(&tok->source);
- }
- if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
- return tok->source.bytes;
- }
- return NULL;
+ return _PyTok_SourceLineView(&tok->source, lineno, length);
}
void
@@ -130,10 +105,7 @@ _PyTokenizer_SetOptions(struct tok_state *tok, int
extra_tokens,
void
_PyTokenizer_ImplyDedents(struct tok_state *tok)
{
- if (tok->indent != 0) {
- tok->pendin = -tok->indent;
- tok->indent = 0;
- }
+ _PyLexer_ImplyDedents(tok);
}
int
@@ -160,11 +132,11 @@ _PyTokenizer_HasTrailingStatement(const struct tok_state
*tok)
int
_PyTokenizer_IsInteractive(const struct tok_state *tok)
{
- return tok->prompt != NULL;
+ return _PyTok_ReaderIsInteractive(tok);
}
void
_PyTokenizer_StopInteractive(struct tok_state *tok)
{
- tok->interactive_underflow = IUNDERFLOW_STOP;
+ _PyTok_ReaderStopInteractive(tok);
}
diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c
deleted file mode 100644
index 523b99dedc6160..00000000000000
--- a/Parser/tokenizer/cursor.c
+++ /dev/null
@@ -1,82 +0,0 @@
-#include "Python.h"
-
-#include "cursor.h"
-
-static void
-set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start,
- _PyTok_Off end)
-{
- cursor->pos = start;
- cursor->line_start = start;
- cursor->line_end = end;
- cursor->lineno = lineno;
-}
-
-int
-_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
-{
- if (cursor->source == NULL) {
- PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
- return -1;
- }
- const _PyTok_SourceText *source = cursor->source;
- if (lineno > 0 && cursor->lineno == lineno - 1 &&
- lineno <= source->nlines) {
- _PyTok_Off start = cursor->line_end;
- _PyTok_Off end = source->base_offset + source->len;
- if (lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- set_line(cursor, lineno, start, end);
- return 0;
- }
-
- _PyTok_Line line;
- if (_PyTok_SourceLine(source, lineno, &line) < 0) {
- return -1;
- }
- set_line(cursor, lineno, line.start, line.end);
- return 0;
-}
-
-int
-_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
-{
- if (cursor->source == NULL) {
- PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
- return -1;
- }
- const _PyTok_SourceText *source = cursor->source;
- int stays_on_line = cursor->lineno > 0 &&
- offset >= cursor->line_start && offset < cursor->line_end;
- if (!stays_on_line && cursor->lineno > 0 &&
- offset == cursor->line_end &&
- offset - source->base_offset == source->len &&
- (source->len == 0 || source->bytes[source->len - 1] != '\n')) {
- stays_on_line = 1;
- }
- if (stays_on_line) {
- cursor->pos = offset;
- return 0;
- }
-
- _PyTok_Loc loc;
- if (_PyTok_SourceLocation(
- source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) {
- return -1;
- }
- _PyTok_Off start = offset - loc.byte_col;
- _PyTok_Off end = source->base_offset + source->len;
- if (loc.lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- set_line(cursor, loc.lineno, start, end);
- cursor->pos = offset;
- return 0;
-}
diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h
deleted file mode 100644
index 18e404251316f0..00000000000000
--- a/Parser/tokenizer/cursor.h
+++ /dev/null
@@ -1,73 +0,0 @@
-#ifndef Py_TOKENIZER_CURSOR_H
-#define Py_TOKENIZER_CURSOR_H
-
-#include "source.h"
-
-typedef struct {
- /* The source must remain initialized at this address while in use. */
- const _PyTok_SourceText *source;
- _PyTok_Off pos;
- _PyTok_Off line_start;
- _PyTok_Off line_end;
- int lineno;
-} _PyTok_Cursor;
-
-/* Move to the start of a 1-based line. Both setters preserve the cursor on
- error. */
-PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int);
-/* Move to an offset. A line boundary selects the following line. */
-PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
-
-static inline void
-_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
-{
- _PyTok_Off base = source != NULL ? source->base_offset : 0;
- *cursor = (_PyTok_Cursor){
- .source = source,
- .pos = base,
- .line_start = base,
- .line_end = base,
- };
-}
-
-/* Read one byte from the current line, including its terminating newline.
- EOF marks the line boundary, not necessarily the end of the source. It is
- also returned if advancing would make the byte column unrepresentable. */
-static inline int
-_PyTok_CursorAdvance(_PyTok_Cursor *cursor)
-{
- assert(cursor->source != NULL);
- assert(cursor->pos >= cursor->line_start);
- assert(cursor->pos <= cursor->line_end);
- assert(cursor->line_start >= cursor->source->base_offset);
- assert(cursor->line_end - cursor->source->base_offset <=
cursor->source->len);
- if (cursor->pos >= cursor->line_end) {
- return EOF;
- }
- if (cursor->pos - cursor->line_start >= INT_MAX) {
- return EOF;
- }
- return Py_CHARMASK(cursor->source->bytes[
- cursor->pos++ - cursor->source->base_offset]);
-}
-
-/* Return the byte at a nonnegative distance within the current line, or EOF
- if the distance reaches or crosses the line boundary. */
-static inline int
-_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
-{
- assert(cursor->source != NULL);
- assert(cursor->pos >= cursor->line_start);
- assert(cursor->pos <= cursor->line_end);
- assert(cursor->line_start >= cursor->source->base_offset);
- assert(cursor->line_end - cursor->source->base_offset <=
cursor->source->len);
- assert(distance >= 0);
- if (distance < 0 ||
- distance >= cursor->line_end - cursor->pos) {
- return EOF;
- }
- return Py_CHARMASK(cursor->source->bytes[
- cursor->pos - cursor->source->base_offset + distance]);
-}
-
-#endif
diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c
index be7442af55c4e3..69c3bb371add96 100644
--- a/Parser/tokenizer/decoder.c
+++ b/Parser/tokenizer/decoder.c
@@ -103,7 +103,9 @@ normalize_newlines_into(char *result, const char *data,
Py_ssize_t len,
}
result[write] = '\0';
*out_len = write;
- *implicit_newline = implicit;
+ if (implicit_newline != NULL) {
+ *implicit_newline = implicit;
+ }
}
char *
@@ -262,7 +264,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const
_PyTok_Chunk *first,
end_col--;
}
_PyTokenizer_syntaxerror_at(
- tok, line_data, 0, cookie_line, 0, end_col, "encoding problem: %s
with BOM", cookie);
+ tok, line_data, 0, cookie_line, 0, end_col,
+ "encoding problem: %s with BOM", cookie);
PyMem_Free(cookie);
return _PYTOK_ENCODING_ERROR;
}
@@ -420,10 +423,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char
*input, int utf8_only,
if (stored < 0) {
return -1;
}
- tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)"";
if (!utf8_only &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
- !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) {
+ !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok,
1)) {
return -1;
}
return 0;
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index d0ada5ac1131a7..0d3ea85109ec49 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -137,10 +137,6 @@ _PyTokenizer_indenterror(struct tok_state *tok)
int
_PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int
first_invalid_escape_char)
{
- if (!tok->report_warnings) {
- return 0;
- }
-
PyObject *msg = PyUnicode_FromFormat(
"\"\\%c\" is an invalid escape sequence. "
"Such sequences will not work in the future. "
@@ -226,10 +222,6 @@ _PyTokenizer_raise_init_error(PyObject *filename)
int
_PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char
*format, ...)
{
- if (!tok->report_warnings) {
- return 0;
- }
-
PyObject *errmsg;
va_list vargs;
va_start(vargs, format);
@@ -341,12 +333,9 @@ _PyTokenizer_ensure_utf8(const char *line, struct
tok_state *tok, int lineno)
}
}
if (badchar) {
- tok->lineno = lineno;
- tok->line_start = _PyLexer_BufferOffset(tok, line_start);
- tok->cur = _PyLexer_BufferOffset(tok, badchar);
- _PyTokenizer_syntaxerror_known_range(tok,
- (int)(badchar - line_start) + 1,
- (int)(badchar - line_start) + 1,
+ _PyTokenizer_syntaxerror_at(
+ tok, line_start, badchar - line_start + 1, lineno,
+ -1, -1,
"Non-UTF-8 code starting with '\\x%.2x'"
"%s%V on line %i, "
"but no encoding declared; "
@@ -389,12 +378,4 @@ _PyTokenizer_print_escape(FILE *f, const char *s,
Py_ssize_t size)
}
putc('"', f);
}
-
-void
-_PyTokenizer_tok_dump(int type, char *start, char *end)
-{
- fprintf(stderr, "%s", _PyParser_TokenNames[type]);
- if (type == NAME || type == NUMBER || type == STRING || type == OP)
- fprintf(stderr, "(%.*s)", (int)(end - start), start);
-}
#endif
diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h
index 24f2d0cff1effb..f33d8f55ed1d17 100644
--- a/Parser/tokenizer/helpers.h
+++ b/Parser/tokenizer/helpers.h
@@ -5,20 +5,24 @@
#include "../lexer/state.h"
-int _PyTokenizer_syntaxerror_at(struct tok_state *, const char *,
- Py_ssize_t, int, int, int, const char *, ...);
int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
+/* Positive range columns are 1-based byte columns. A start column of -1
+ derives the character column from the reporting cursor; an end column of
+ -1 uses the start column. */
int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int
col_offset, int end_col_offset, const char *format, ...);
+int _PyTokenizer_syntaxerror_at(
+ struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset,
+ int lineno, int col_offset, int end_col_offset, const char *format, ...);
int _PyTokenizer_indenterror(struct tok_state *tok);
int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int
first_invalid_escape_char);
int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const
char *format, ...);
+
void _PyTokenizer_raise_init_error(PyObject *filename);
int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int
lineno);
#ifdef Py_DEBUG
void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size);
-void _PyTokenizer_tok_dump(int type, char *start, char *end);
#endif
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 77bfecc0cf12f7..c003deba18dd8f 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -1,11 +1,11 @@
#include "Python.h"
#include "pycore_fileutils.h"
+#include "pycore_pystate.h"
#include "errcode.h"
#include "helpers.h"
#include "reader.h"
#include "reader_internal.h"
-#include "../lexer/lexer.h"
#include "../lexer/state.h"
#ifdef HAVE_UNISTD_H
@@ -139,10 +139,10 @@ chunk_is_line(const _PyTok_Chunk *chunk)
static _PyTok_ReadResult
next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
{
- int lineno = tok->lineno + 1;
- if (lineno > tok->source.nlines) {
+ if (tok->lineno >= tok->source.nlines) {
return _PYTOK_READ_EOF;
}
+ int lineno = tok->lineno + 1;
const char *start = _PyLexer_BufferPointer(tok, tok->inp);
const char *newline = memchr(
start, '\n', tok->source.bytes + tok->source.len - start);
@@ -208,7 +208,6 @@ initialize_file(struct tok_state *tok)
if (result != _PYTOK_READ_LINE) {
return -1;
}
- reader->prefetched_count = 1;
Py_ssize_t bom_len;
_PyTok_EncodingResult detection = _PyTok_DetectEncoding(
tok, &reader->prefetched_lines[0], NULL, 0, &bom_len);
@@ -226,16 +225,13 @@ initialize_file(struct tok_state *tok)
reader->prefetched_lines[0].data = first;
reader->prefetched_lines[0].ownership = _PYTOK_CHUNK_PYMEM;
result = read_file_line(tok, &reader->prefetched_lines[1]);
- if (result == _PYTOK_READ_LINE) {
- reader->prefetched_count = 2;
- }
- else if (result == _PYTOK_READ_EOF) {
+ if (result == _PYTOK_READ_EOF) {
reader->file_eof = 1;
}
- else {
+ else if (result != _PYTOK_READ_LINE) {
return -1;
}
- _PyTok_Chunk *second = reader->prefetched_count == 2
+ _PyTok_Chunk *second = reader->prefetched_lines[1].data != NULL
? &reader->prefetched_lines[1] : NULL;
detection = _PyTok_DetectEncoding(
tok, &reader->prefetched_lines[0], second, 1, &bom_len);
@@ -305,10 +301,13 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk)
return _PYTOK_READ_LINE;
}
_PyTok_Chunk input = {0};
- if (reader->prefetched_index < reader->prefetched_count) {
- input = reader->prefetched_lines[reader->prefetched_index];
- reader->prefetched_lines[reader->prefetched_index++] =
- (_PyTok_Chunk){0};
+ if (reader->prefetched_lines[0].data != NULL) {
+ input = reader->prefetched_lines[0];
+ reader->prefetched_lines[0] = (_PyTok_Chunk){0};
+ }
+ else if (reader->prefetched_lines[1].data != NULL) {
+ input = reader->prefetched_lines[1];
+ reader->prefetched_lines[1] = (_PyTok_Chunk){0};
}
else if (!reader->file_eof) {
_PyTok_ReadResult result = read_file_line(tok, &input);
@@ -477,13 +476,13 @@ static _PyTok_ReadResult
next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
{
_PyTok_Reader *reader = tok->reader;
- if (tok->interactive_underflow == IUNDERFLOW_STOP) {
+ if (reader->stop_interactive) {
return _PYTOK_READ_STOPPED;
}
char *input = PyOS_Readline(
- tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt);
+ tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt);
if (reader->nextprompt != NULL) {
- tok->prompt = reader->nextprompt;
+ reader->prompt = reader->nextprompt;
}
if (input == NULL) {
return _PYTOK_READ_INTERRUPT;
@@ -506,7 +505,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
}
chunk->data = _PyTok_NormalizeNewlines(
decoded.data, decoded.len, 0, 0,
- &chunk->len, &chunk->implicit_newline);
+ &chunk->len, NULL);
_PyTok_ChunkClear(&decoded);
if (chunk->data == NULL) {
PyErr_NoMemory();
@@ -517,6 +516,32 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk
*chunk)
return _PYTOK_READ_LINE;
}
+int
+_PyTok_ReaderIsInteractive(const struct tok_state *tok)
+{
+ return tok->reader->kind == _PYTOK_READER_INTERACTIVE;
+}
+
+const char *
+_PyTokenizer_RetainedSource(const struct tok_state *tok)
+{
+ if (tok->reader->kind == _PYTOK_READER_PREPARED) {
+ return _PyTok_SourceData(&tok->source);
+ }
+ if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
+ return tok->source.bytes;
+ }
+ return NULL;
+}
+
+void
+_PyTok_ReaderStopInteractive(struct tok_state *tok)
+{
+ if (_PyTok_ReaderIsInteractive(tok)) {
+ tok->reader->stop_interactive = 1;
+ }
+}
+
static _PyTok_ReadResult
reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
{
@@ -581,12 +606,13 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
}
return 0;
}
-
- Py_ssize_t scan_len = chunk.len;
- if (kind == _PYTOK_READER_INTERACTIVE &&
- chunk.implicit_newline) {
- scan_len--;
+ if (tok->lineno == INT_MAX) {
+ PyErr_SetString(PyExc_OverflowError, "too many tokenizer source
lines");
+ tok->done = E_ERROR;
+ _PyTok_ChunkClear(&chunk);
+ return 0;
}
+
if (!prepared) {
if (streaming && reset_buffer) {
reset_streaming_buffer(tok);
@@ -606,11 +632,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
tok->line_start = tok->buf_offset;
tok->start = -1;
}
- tok->inp = source_start + scan_len;
- }
- if (tok->fp_interactive) {
- tok->interactive_src_start = tok->source.bytes;
- tok->interactive_src_end = tok->source.bytes + tok->source.len;
+ tok->inp = source_start + chunk.len;
}
if (prepared) {
if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) {
@@ -635,10 +657,21 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
static struct tok_state *
tokenizer_new_with_reader(_PyTok_ReaderKind kind)
{
- struct tok_state *tok = _PyTokenizer_tok_new();
+ struct tok_state *tok = PyMem_Calloc(1, sizeof(*tok));
if (tok == NULL) {
+ PyErr_NoMemory();
return NULL;
}
+ tok->start = tok->line_start = -1;
+ _PyTok_SourceInit(&tok->source);
+ tok->done = E_OK;
+ tok->layout.at_bol = 1;
+ tok->start_loc = (_PyTok_Loc){-1, -1};
+ tok->ftstring_stack = tok->ftstring_stack_inline;
+ tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY;
+#ifdef Py_DEBUG
+ tok->debug = _Py_GetConfig()->parser_debug;
+#endif
tok->reader = PyMem_Calloc(1, sizeof(*tok->reader));
if (tok->reader == NULL) {
PyErr_NoMemory();
@@ -716,7 +749,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding,
return NULL;
}
tok->fp = fp;
- tok->prompt = ps1;
+ tok->reader->prompt = ps1;
tok->reader->nextprompt = ps2;
return tok;
}
@@ -770,13 +803,10 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject
*filename)
_PyTokenizer_Free(tok);
return NULL;
}
- /* Reporting a warning here could recursively ask for the encoding. */
- tok->report_warnings = 0;
- while (tok->lineno < 2 && tok->done == E_OK) {
- struct token token;
- _PyToken_Init(&token);
- _PyTokenizer_Get(tok, &token);
- _PyToken_Free(&token);
+ if (initialize_file(tok) < 0) {
+ fclose(fp);
+ _PyTokenizer_Free(tok);
+ return NULL;
}
fclose(fp);
char *encoding = tok->encoding == NULL
diff --git a/Parser/tokenizer/reader.h b/Parser/tokenizer/reader.h
index c27bc2aa3fb819..2913e52b9d563b 100644
--- a/Parser/tokenizer/reader.h
+++ b/Parser/tokenizer/reader.h
@@ -5,5 +5,7 @@ struct tok_state;
void _PyTok_ReaderFree(struct tok_state *);
int _PyTok_ReaderUnderflow(struct tok_state *);
+int _PyTok_ReaderIsInteractive(const struct tok_state *);
+void _PyTok_ReaderStopInteractive(struct tok_state *);
#endif
diff --git a/Parser/tokenizer/reader_internal.h
b/Parser/tokenizer/reader_internal.h
index 5071df35c142d3..390b6ed9d8581a 100644
--- a/Parser/tokenizer/reader_internal.h
+++ b/Parser/tokenizer/reader_internal.h
@@ -32,33 +32,32 @@ typedef enum {
typedef struct {
char *data;
- Py_ssize_t len;
- int implicit_newline;
PyObject *owner;
+ Py_ssize_t len;
_PyTok_ChunkOwnership ownership;
+ unsigned char implicit_newline;
} _PyTok_Chunk;
typedef struct _PyTok_Reader {
- _PyTok_ReaderKind kind;
PyObject *readline;
PyObject *decoder;
+ const char *prompt;
const char *nextprompt;
char *file_buffer;
Py_ssize_t file_buffer_cap;
_PyTok_Chunk prefetched_lines[2];
- int prefetched_index;
- int prefetched_count;
char *decoded;
Py_ssize_t decoded_pos;
Py_ssize_t decoded_len;
Py_ssize_t decoded_cap;
- int decoded_tail_is_implicit;
-
- int file_initialized;
- int file_eof;
- int decoder_finalized;
+ _PyTok_ReaderKind kind;
+ unsigned char decoded_tail_is_implicit;
+ unsigned char file_initialized;
+ unsigned char file_eof;
+ unsigned char decoder_finalized;
+ unsigned char stop_interactive;
} _PyTok_Reader;
struct tok_state;
diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c
index 2f2aaf2589246d..d69eab93923e81 100644
--- a/Parser/tokenizer/source.c
+++ b/Parser/tokenizer/source.c
@@ -2,8 +2,6 @@
#include "source.h"
-#define LINE_CHECKPOINT_INTERVAL 256
-
void
_PyTok_SourceInit(_PyTok_SourceText *source)
{
@@ -14,7 +12,6 @@ void
_PyTok_SourceClear(_PyTok_SourceText *source)
{
PyMem_Free(source->bytes);
- PyMem_Free(source->line_checkpoints);
PyMem_Free(source->implicit_lines);
_PyTok_SourceInit(source);
}
@@ -74,34 +71,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
return 0;
}
-static int
-reserve_checkpoints(_PyTok_SourceText *source, int needed)
-{
- if (needed <= source->checkpoints_cap) {
- return 0;
- }
- int cap;
- if (source->checkpoints_cap == 0) {
- cap = 16;
- }
- else if (source->checkpoints_cap <= INT_MAX / 2) {
- cap = source->checkpoints_cap * 2;
- }
- else {
- PyErr_NoMemory();
- return -1;
- }
- _PyTok_Off *checkpoints = source->line_checkpoints;
- PyMem_Resize(checkpoints, _PyTok_Off, cap);
- if (checkpoints == NULL) {
- PyErr_NoMemory();
- return -1;
- }
- source->line_checkpoints = checkpoints;
- source->checkpoints_cap = cap;
- return 0;
-}
-
static int
reserve_implicit_lines(_PyTok_SourceText *source, int nlines)
{
@@ -165,11 +134,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const
char *bytes,
return -1;
}
int nlines = source->nlines + 1;
- int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0;
- int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
- if ((checkpoint &&
- reserve_checkpoints(source, checkpoint_count) < 0) ||
- (implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
+ if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
reserve_bytes(source, source->len + len + 1) < 0) {
return -1;
}
@@ -178,10 +143,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const
char *bytes,
memcpy(source->bytes + start, bytes, len);
source->len += len;
source->bytes[source->len] = '\0';
- if (checkpoint) {
- source->line_checkpoints[checkpoint_count - 1] =
- source->base_offset + start;
- }
if (implicit_newline) {
source->implicit_lines[(nlines - 1) / 8] |=
(unsigned char)(1U << ((nlines - 1) & 7));
@@ -191,16 +152,23 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const
char *bytes,
}
const char *
-_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
+_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno,
Py_ssize_t *len)
{
- if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset ||
- span.end - source->base_offset > source->len || len == NULL) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span");
- return NULL;
+ assert(len != NULL);
+ const char *line = _PyTok_SourceData(source);
+ const char *end = line + source->len;
+ while (lineno > 1) {
+ const char *newline = memchr(line, '\n', end - line);
+ if (newline == NULL) {
+ break;
+ }
+ line = newline + 1;
+ lineno--;
}
- *len = span.end - span.start;
- return _PyTok_SourceData(source) + (span.start - source->base_offset);
+ const char *newline = memchr(line, '\n', end - line);
+ *len = (newline != NULL ? newline : end) - line;
+ return line;
}
int
@@ -213,124 +181,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText
*source, int lineno)
return (source->implicit_lines[(lineno - 1) / 8] >>
((lineno - 1) & 7)) & 1;
}
-
-static int
-source_ends_in_newline(const _PyTok_SourceText *source)
-{
- return source->len > 0 && source->bytes[source->len - 1] == '\n';
-}
-
-static int
-eof_lineno(const _PyTok_SourceText *source)
-{
- if (source->nlines == 0) {
- return 1;
- }
- return source->nlines + source_ends_in_newline(source);
-}
-
-int
-_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
- _PyTok_Line *line)
-{
- if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
- return -1;
- }
- if (lineno > source->nlines) {
- *line = (_PyTok_Line){
- .start = source->base_offset + source->len,
- .end = source->base_offset + source->len,
- };
- return 0;
- }
-
- int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL;
- int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
- _PyTok_Off start = source->line_checkpoints[checkpoint];
- while (current < lineno) {
- start = _PyTok_SourceFindLineEnd(source, start);
- if (start < 0) {
- return -1;
- }
- current++;
- }
- _PyTok_Off end = source->base_offset + source->len;
- if (lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- *line = (_PyTok_Line){
- .start = start,
- .end = end,
- .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno),
- .contains_nul = memchr(
- source->bytes + (start - source->base_offset),
- 0, end - start) != NULL,
- };
- return 0;
-}
-
-int
-_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset,
- _PyTok_Affinity affinity, _PyTok_Loc *loc)
-{
- if (offset < source->base_offset ||
- offset - source->base_offset > source->len || loc == NULL ||
- (affinity != _PYTOK_AFFINITY_LEFT &&
- affinity != _PYTOK_AFFINITY_RIGHT)) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset");
- return -1;
- }
- if (source->nlines == 0 ||
- (offset - source->base_offset == source->len &&
- source_ends_in_newline(source) &&
- affinity == _PYTOK_AFFINITY_RIGHT)) {
- *loc = (_PyTok_Loc){eof_lineno(source), 0};
- return 0;
- }
-
- _PyTok_Off key = offset;
- if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) {
- key--;
- }
- int low = 0;
- int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
- while (low < high) {
- int middle = low + (high - low) / 2;
- if (source->line_checkpoints[middle] <= key) {
- low = middle + 1;
- }
- else {
- high = middle;
- }
- }
- int checkpoint = low - 1;
- if (checkpoint < 0) {
- PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line
index");
- return -1;
- }
- int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
- _PyTok_Off start = source->line_checkpoints[checkpoint];
- while (lineno < source->nlines) {
- _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- if (offset < end ||
- (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) {
- break;
- }
- start = end;
- lineno++;
- }
- _PyTok_Off byte_col = offset - start;
- if (byte_col > INT_MAX) {
- PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large");
- return -1;
- }
- *loc = (_PyTok_Loc){lineno, (int)byte_col};
- return 0;
-}
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index 2f74ed8b1f4fab..9576419aade1d4 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -5,28 +5,13 @@
#include "types.h"
-typedef enum {
- _PYTOK_AFFINITY_LEFT,
- _PYTOK_AFFINITY_RIGHT,
-} _PyTok_Affinity;
-
-/* The half-open range includes the terminating newline when present. */
-typedef struct {
- _PyTok_Off start;
- _PyTok_Off end;
- unsigned implicit_newline : 1;
- unsigned contains_nul : 1;
-} _PyTok_Line;
-
typedef struct {
char *bytes;
_PyTok_Off base_offset;
_PyTok_Off len;
_PyTok_Off cap;
- _PyTok_Off *line_checkpoints;
unsigned char *implicit_lines;
int nlines;
- int checkpoints_cap;
Py_ssize_t implicit_cap;
} _PyTok_SourceText;
@@ -37,9 +22,9 @@ _PyTok_SourceData(const _PyTok_SourceText *source)
}
PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
-/* Clear invalidates all cursors, spans, and views for the source. */
+/* Clear invalidates all spans and views for the source. */
PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
-/* Discard the retained window and invalidate its cursors, spans, and views.
+/* Discard the retained window and invalidate its spans and views.
Keep its allocation and advance the logical base to the end of the window.
*/
PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
/* Append one nonempty logical line and return its start offset. The input may
@@ -49,39 +34,14 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
_PyTok_SourceText *source, const char *bytes, Py_ssize_t len,
int implicit_newline);
-/* The returned view is invalidated by SourceAppendLine and SourceClear. */
-PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
- const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
-/* Look up a 1-based line in the retained window. Empty and newline-terminated
- sources have an empty virtual line at EOF. */
-PyAPI_FUNC(int) _PyTok_SourceLine(
- const _PyTok_SourceText *, int, _PyTok_Line *);
+/* Return borrowed bytes excluding '\n', writing the byte length to *len.
+ Line numbers are 1-based and clamp to the first or final line; a trailing
+ '\n' adds an empty final line. The view need not be NUL-terminated.
+ This does not set an exception. Append, discard, and clear invalidate the
view. */
+PyAPI_FUNC(const char *) _PyTok_SourceLineView(
+ const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len);
/* Return false for invalid line numbers and the virtual EOF line. */
PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
const _PyTok_SourceText *, int);
-/* At a line boundary, left affinity selects the preceding line at its end;
- right affinity selects the following line at byte column zero. */
-PyAPI_FUNC(int) _PyTok_SourceLocation(
- const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
-
-static inline _PyTok_Off
-_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
-{
- if (source->bytes == NULL || start < source->base_offset ||
- start - source->base_offset >= source->len) {
- PyErr_SetString(PyExc_SystemError,
- "corrupt tokenizer source line index");
- return -1;
- }
- _PyTok_Off relative_start = start - source->base_offset;
- const char *newline = memchr(
- source->bytes + relative_start, '\n', source->len - relative_start);
- if (newline == NULL) {
- PyErr_SetString(PyExc_SystemError,
- "corrupt tokenizer source line index");
- return -1;
- }
- return source->base_offset + (newline - source->bytes) + 1;
-}
#endif
diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h
index e9229d12087162..82a84830d7cbfe 100644
--- a/Parser/tokenizer/tokenizer.h
+++ b/Parser/tokenizer/tokenizer.h
@@ -28,8 +28,17 @@ typedef struct {
int at_eof;
} _PyToken_View;
+/* Supplemental source context for a terminal error. location is the reporting
+ cursor, independent of the scanner cursor; lineno == 0 means absent.
+ The text span may cover multiple physical lines. */
+typedef struct {
+ _PyTok_Loc location;
+ _PyTok_Span text_span;
+} _PyTokenizer_Diagnostic;
+
typedef struct {
int status;
+ _PyTokenizer_Diagnostic diagnostic;
_PyTok_Loc location;
_PyTok_Off cursor;
_PyTok_Span input_span;
@@ -98,6 +107,4 @@ struct tok_state *_PyTokenizer_FromFile(
An exception is set on error. */
char *_PyTokenizer_FindEncodingFilename(int, PyObject *);
-#define tok_dump _Py_tok_dump
-
#endif /* !Py_TOKENIZER_H */
diff --git a/Tools/peg_generator/pegen/build.py
b/Tools/peg_generator/pegen/build.py
index 1dc33520e5387d..ce079adafcf4c7 100644
--- a/Tools/peg_generator/pegen/build.py
+++ b/Tools/peg_generator/pegen/build.py
@@ -125,6 +125,7 @@ def compile_c_extension(
str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"),
+ str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
