Author: Carl Friedrich Bolz-Tereick <[email protected]>
Branch: pyparser-improvements-2
Changeset: r94275:52f6bfe8f3cd
Date: 2018-04-07 14:25 +0200
http://bitbucket.org/pypy/pypy/changeset/52f6bfe8f3cd/
Log: fix more offsets
diff --git a/pypy/interpreter/pyparser/pytokenizer.py
b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -98,7 +98,7 @@
if not line:
raise TokenError(
"EOF while scanning triple-quoted string literal",
- strstart[2], strstart[0], strstart[1]+1,
+ strstart[2], strstart[0], strstart[1],
token_list, lnum-1)
endmatch = endDFA.recognize(line)
if endmatch >= 0:
@@ -248,7 +248,7 @@
start = pos
if start<max and line[start] in single_quoted:
raise TokenError("EOL while scanning string literal",
- line, lnum, start+1, token_list)
+ line, lnum, start, token_list)
tok = (tokens.ERRORTOKEN, line[pos], lnum, pos, line)
token_list.append(tok)
last_comment = ''
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py
b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -81,11 +81,11 @@
exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value
assert exc.msg == "EOL while scanning string literal"
assert exc.lineno == 1
- assert exc.offset == 5
+ assert exc.offset == 4
exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value
assert exc.msg == "EOF while scanning triple-quoted string literal"
assert exc.lineno == 1
- assert exc.offset == 5
+ assert exc.offset == 4
assert exc.lastlineno == 3
for input in ("())", "(()", "((", "))"):
py.test.raises(SyntaxError, parse, input)
diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py
b/pypy/interpreter/pyparser/test/test_pytokenizer.py
--- a/pypy/interpreter/pyparser/test/test_pytokenizer.py
+++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py
@@ -6,11 +6,14 @@
def tokenize(s):
return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0)
-def check_token_error(s, msg, pos=-1):
+def check_token_error(s, msg=None, pos=-1, line=-1):
error = pytest.raises(TokenError, tokenize, s)
- assert error.value.msg == msg
+ if msg is not None:
+ assert error.value.msg == msg
if pos != -1:
assert error.value.offset == pos
+ if line != -1:
+ assert error.value.lineno == line
class TestTokenizer(object):
@@ -52,3 +55,9 @@
def test_unknown_char(self):
check_token_error("?", "Unknown character", 0)
+
+ def test_eol_string(self):
+ check_token_error("x = 'a", pos=4, line=1)
+
+ def test_eof_triple_quoted(self):
+ check_token_error("'''", pos=0, line=1)
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
