Author: Carl Friedrich Bolz-Tereick <cfb...@gmx.de> Branch: pyparser-improvements-2 Changeset: r94275:52f6bfe8f3cd Date: 2018-04-07 14:25 +0200 http://bitbucket.org/pypy/pypy/changeset/52f6bfe8f3cd/
Log: fix more offsets diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -98,7 +98,7 @@ if not line: raise TokenError( "EOF while scanning triple-quoted string literal", - strstart[2], strstart[0], strstart[1]+1, + strstart[2], strstart[0], strstart[1], token_list, lnum-1) endmatch = endDFA.recognize(line) if endmatch >= 0: @@ -248,7 +248,7 @@ start = pos if start<max and line[start] in single_quoted: raise TokenError("EOL while scanning string literal", - line, lnum, start+1, token_list) + line, lnum, start, token_list) tok = (tokens.ERRORTOKEN, line[pos], lnum, pos, line) token_list.append(tok) last_comment = '' diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -81,11 +81,11 @@ exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value assert exc.msg == "EOL while scanning string literal" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 4 exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value assert exc.msg == "EOF while scanning triple-quoted string literal" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 4 assert exc.lastlineno == 3 for input in ("())", "(()", "((", "))"): py.test.raises(SyntaxError, parse, input) diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py --- a/pypy/interpreter/pyparser/test/test_pytokenizer.py +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -6,11 +6,14 @@ def tokenize(s): return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) -def check_token_error(s, msg, pos=-1): +def check_token_error(s, msg=None, pos=-1, line=-1): error = pytest.raises(TokenError, tokenize, s) - assert error.value.msg == msg + if msg is not None: + assert error.value.msg == msg if pos != -1: assert error.value.offset == pos + if line != -1: + assert error.value.lineno == line class TestTokenizer(object): @@ -52,3 +55,9 @@ def test_unknown_char(self): check_token_error("?", "Unknown character", 0) + + def test_eol_string(self): + check_token_error("x = 'a", pos=4, line=1) + + def test_eof_triple_quoted(self): + check_token_error("'''", pos=0, line=1) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit