Author: Carl Friedrich Bolz-Tereick <cfb...@gmx.de>
Branch: pyparser-improvements-2
Changeset: r94273:7eaa7ec2554a
Date: 2018-04-07 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/7eaa7ec2554a/

Log:    track matching parenthesis for better errors

diff --git a/pypy/interpreter/pyparser/pytokenizer.py 
b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -73,14 +73,14 @@
         logical line; continuation lines are included.
     """
     token_list = []
-    lnum = parenlev = continued = 0
+    lnum = continued = 0
     namechars = NAMECHARS
     numchars = NUMCHARS
     contstr, needcont = '', 0
     contline = None
     indents = [0]
     last_comment = ''
-    parenlevstart = (0, 0, "")
+    parenstack = []
 
     # make the annotator happy
     endDFA = DUMMY_DFA
@@ -123,7 +123,7 @@
                 contline = contline + line
                 continue
 
-        elif parenlev == 0 and not continued:  # new statement
+        elif not parenstack and not continued:  # new statement
             if not line: break
             column = 0
             while pos < max:                   # measure leading whitespace
@@ -152,10 +152,10 @@
 
         else:                                  # continued statement
             if not line:
-                if parenlev > 0:
-                    lnum1, start1, line1 = parenlevstart
+                if parenstack:
+                    _, lnum1, start1, line1 = parenstack[0]
                     raise TokenError("parenthesis is never closed", line1,
-                                     lnum1, start1 + 1, token_list, lnum)
+                                     lnum1, start1, token_list, lnum)
                 raise TokenError("EOF in multi-line statement", line,
                                  lnum, 0, token_list)
             continued = 0
@@ -180,7 +180,7 @@
                     token_list.append((tokens.NUMBER, token, lnum, start, 
line))
                     last_comment = ''
                 elif initial in '\r\n':
-                    if parenlev <= 0:
+                    if not parenstack:
                         tok = (tokens.NEWLINE, last_comment, lnum, start, line)
                         token_list.append(tok)
                     last_comment = ''
@@ -222,14 +222,20 @@
                     continued = 1
                 else:
                     if initial in '([{':
-                        if parenlev == 0:
-                            parenlevstart = (lnum, start, line)
-                        parenlev = parenlev + 1
+                        parenstack.append((initial, lnum, start, line))
                     elif initial in ')]}':
-                        parenlev = parenlev - 1
-                        if parenlev < 0:
+                        if not parenstack:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum, start + 1, token_list)
+                                             lnum, start, token_list)
+                        opening, lnum1, start1, line1 = parenstack.pop()
+                        if not ((opening == "(" and initial == ")") or
+                                (opening == "[" and initial == "]") or
+                                (opening == "{" and initial == "}")):
+                            raise TokenError(
+                                    "parenthesis '%s' and '%s' don't match" % (
+                                        opening, initial),
+                                    line1, lnum1, start1, token_list,
+                                    lastlineno=lnum)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py 
b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -92,12 +92,12 @@
         exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value
         assert exc.msg == "parenthesis is never closed"
         assert exc.lineno == 1
-        assert exc.offset == 5
+        assert exc.offset == 4
         assert exc.lastlineno == 5
         exc = py.test.raises(SyntaxError, parse, "abc)").value
         assert exc.msg == "unmatched ')'"
         assert exc.lineno == 1
-        assert exc.offset == 4
+        assert exc.offset == 3
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py 
b/pypy/interpreter/pyparser/test/test_pytokenizer.py
--- a/pypy/interpreter/pyparser/test/test_pytokenizer.py
+++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py
@@ -6,9 +6,11 @@
 def tokenize(s):
     return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0)
 
-def check_token_error(s, msg):
+def check_token_error(s, msg, pos=-1):
     error = pytest.raises(TokenError, tokenize, s)
     assert error.value.msg == msg
+    if pos != -1:
+        assert error.value.offset == pos
 
 
 class TestTokenizer(object):
@@ -28,10 +30,22 @@
     def test_error_parenthesis(self):
         for paren in "([{":
             check_token_error(paren + "1 + 2",
-                              "parenthesis is never closed")
+                              "parenthesis is never closed",
+                              0)
 
         for paren in ")]}":
             check_token_error("1 + 2" + paren,
-                              "unmatched '%s'" % (paren, ))
+                              "unmatched '%s'" % (paren, ),
+                              5)
 
+        for i, opening in enumerate("([{"):
+            for j, closing in enumerate(")]}"):
+                if i == j:
+                    continue
+                error = pytest.raises(TokenError, tokenize, opening + "1\n" + 
closing)
+                assert error.value.msg == \
+                        "parenthesis '%s' and '%s' don't match" % (opening, 
closing)
+                assert error.value.offset == 0
+                assert error.value.lineno == 1
+                assert error.value.lastlineno == 2
 
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

Reply via email to