Author: Carl Friedrich Bolz-Tereick <[email protected]> Branch: pyparser-improvements-2 Changeset: r94271:61275f7cf5ea Date: 2018-04-07 13:44 +0200 http://bitbucket.org/pypy/pypy/changeset/61275f7cf5ea/
Log: add at least a very simple test for the tokenizer diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -0,0 +1,19 @@ +from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser.pygram import tokens + +def tokenize(s): + return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) + +class TestTokenizer(object): + + def test_simple(self): + line = "a+1" + tks = tokenize(line) + assert tks == [ + (tokens.NAME, 'a', 1, 0, line), + (tokens.PLUS, '+', 1, 1, line), + (tokens.NUMBER, '1', 1, 2, line), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.ENDMARKER, '', 2, 0, ''), + ] _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
