Author: Carl Friedrich Bolz-Tereick <[email protected]>
Branch: pyparser-improvements-2
Changeset: r94273:7eaa7ec2554a
Date: 2018-04-07 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/7eaa7ec2554a/
Log: track matching parenthesis for better errors
diff --git a/pypy/interpreter/pyparser/pytokenizer.py
b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -73,14 +73,14 @@
logical line; continuation lines are included.
"""
token_list = []
- lnum = parenlev = continued = 0
+ lnum = continued = 0
namechars = NAMECHARS
numchars = NUMCHARS
contstr, needcont = '', 0
contline = None
indents = [0]
last_comment = ''
- parenlevstart = (0, 0, "")
+ parenstack = []
# make the annotator happy
endDFA = DUMMY_DFA
@@ -123,7 +123,7 @@
contline = contline + line
continue
- elif parenlev == 0 and not continued: # new statement
+ elif not parenstack and not continued: # new statement
if not line: break
column = 0
while pos < max: # measure leading whitespace
@@ -152,10 +152,10 @@
else: # continued statement
if not line:
- if parenlev > 0:
- lnum1, start1, line1 = parenlevstart
+ if parenstack:
+ _, lnum1, start1, line1 = parenstack[0]
raise TokenError("parenthesis is never closed", line1,
- lnum1, start1 + 1, token_list, lnum)
+ lnum1, start1, token_list, lnum)
raise TokenError("EOF in multi-line statement", line,
lnum, 0, token_list)
continued = 0
@@ -180,7 +180,7 @@
token_list.append((tokens.NUMBER, token, lnum, start,
line))
last_comment = ''
elif initial in '\r\n':
- if parenlev <= 0:
+ if not parenstack:
tok = (tokens.NEWLINE, last_comment, lnum, start, line)
token_list.append(tok)
last_comment = ''
@@ -222,14 +222,20 @@
continued = 1
else:
if initial in '([{':
- if parenlev == 0:
- parenlevstart = (lnum, start, line)
- parenlev = parenlev + 1
+ parenstack.append((initial, lnum, start, line))
elif initial in ')]}':
- parenlev = parenlev - 1
- if parenlev < 0:
+ if not parenstack:
raise TokenError("unmatched '%s'" % initial, line,
- lnum, start + 1, token_list)
+ lnum, start, token_list)
+ opening, lnum1, start1, line1 = parenstack.pop()
+ if not ((opening == "(" and initial == ")") or
+ (opening == "[" and initial == "]") or
+ (opening == "{" and initial == "}")):
+ raise TokenError(
+ "parenthesis '%s' and '%s' don't match" % (
+ opening, initial),
+ line1, lnum1, start1, token_list,
+ lastlineno=lnum)
if token in python_opmap:
punct = python_opmap[token]
else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py
b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -92,12 +92,12 @@
exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value
assert exc.msg == "parenthesis is never closed"
assert exc.lineno == 1
- assert exc.offset == 5
+ assert exc.offset == 4
assert exc.lastlineno == 5
exc = py.test.raises(SyntaxError, parse, "abc)").value
assert exc.msg == "unmatched ')'"
assert exc.lineno == 1
- assert exc.offset == 4
+ assert exc.offset == 3
def test_is(self):
self.parse("x is y")
diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py
b/pypy/interpreter/pyparser/test/test_pytokenizer.py
--- a/pypy/interpreter/pyparser/test/test_pytokenizer.py
+++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py
@@ -6,9 +6,11 @@
def tokenize(s):
return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0)
-def check_token_error(s, msg):
+def check_token_error(s, msg, pos=-1):
error = pytest.raises(TokenError, tokenize, s)
assert error.value.msg == msg
+ if pos != -1:
+ assert error.value.offset == pos
class TestTokenizer(object):
@@ -28,10 +30,22 @@
def test_error_parenthesis(self):
for paren in "([{":
check_token_error(paren + "1 + 2",
- "parenthesis is never closed")
+ "parenthesis is never closed",
+ 0)
for paren in ")]}":
check_token_error("1 + 2" + paren,
- "unmatched '%s'" % (paren, ))
+ "unmatched '%s'" % (paren, ),
+ 5)
+ for i, opening in enumerate("([{"):
+ for j, closing in enumerate(")]}"):
+ if i == j:
+ continue
+ error = pytest.raises(TokenError, tokenize, opening + "1\n" +
closing)
+ assert error.value.msg == \
+ "parenthesis '%s' and '%s' don't match" % (opening,
closing)
+ assert error.value.offset == 0
+ assert error.value.lineno == 1
+ assert error.value.lastlineno == 2
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
