3 new commits in py:
https://bitbucket.org/hpk42/py/changeset/e6cf2e2fee51/ changeset: e6cf2e2fee51 user: hpk42 date: 2012-10-25 09:44:01 summary: convert terminal write-data to unicode if it is not a string/bytes affected #: 6 files diff -r 7e29aba2f9791e942b87149cbb0c4812a7bf2f0a -r e6cf2e2fee5156e174a547beeb7d827d723a944b CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,8 @@ - fix a doc link to bug tracker - try to make terminal.write() printing more robust against unicodeencode/decode problems, amend according test +- introduce py.builtin.text and py.builtin.bytes + to point to respective str/unicode (py2) and bytes/str (py3) types Changes between 1.4.9 and 1.4.10 ================================================== diff -r 7e29aba2f9791e942b87149cbb0c4812a7bf2f0a -r e6cf2e2fee5156e174a547beeb7d827d723a944b py/__init__.py --- a/py/__init__.py +++ b/py/__init__.py @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = '1.4.11.dev2' +__version__ = '1.4.11.dev4' from py import _apipkg @@ -104,6 +104,8 @@ 'builtins' : '._builtin:builtins', 'execfile' : '._builtin:execfile', 'callable' : '._builtin:callable', + 'bytes' : '._builtin:bytes', + 'text' : '._builtin:text', }, # input-output helping diff -r 7e29aba2f9791e942b87149cbb0c4812a7bf2f0a -r e6cf2e2fee5156e174a547beeb7d827d723a944b py/_builtin.py --- a/py/_builtin.py +++ b/py/_builtin.py @@ -128,6 +128,10 @@ def _istext(x): return isinstance(x, str) + text = str + bytes = bytes + + def _getimself(function): return getattr(function, '__self__', None) @@ -160,6 +164,8 @@ import __builtin__ as builtins _totext = unicode _basestring = basestring + text = unicode + bytes = str execfile = execfile callable = callable def _isbytes(x): diff -r 7e29aba2f9791e942b87149cbb0c4812a7bf2f0a -r e6cf2e2fee5156e174a547beeb7d827d723a944b py/_io/terminalwriter.py --- a/py/_io/terminalwriter.py +++ b/py/_io/terminalwriter.py @@ -8,6 +8,7 @@ import sys, os import py py3k = sys.version_info[0] >= 3 +from py.builtin import text, bytes win32_and_ctypes = False if sys.platform == "win32": @@ -163,6 +164,8 @@ def write(self, msg, **kw): if msg: + if not isinstance(msg, (bytes, text)): + msg = text(msg) if self.hasmarkup and kw: markupmsg = self.markup(msg, **kw) else: diff -r 7e29aba2f9791e942b87149cbb0c4812a7bf2f0a -r e6cf2e2fee5156e174a547beeb7d827d723a944b setup.py --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ name='py', description='library with cross-python path, ini-parsing, io, code, log facilities', long_description = open('README.txt').read(), - version='1.4.11.dev2', + version='1.4.11.dev4', url='http://pylib.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff -r 7e29aba2f9791e942b87149cbb0c4812a7bf2f0a -r e6cf2e2fee5156e174a547beeb7d827d723a944b testing/root/test_builtin.py --- a/testing/root/test_builtin.py +++ b/testing/root/test_builtin.py @@ -133,6 +133,14 @@ def test_totext(): py.builtin._totext("hello", "UTF-8") +def test_bytes_text(): + if sys.version_info[0] < 3: + assert py.builtin.text == unicode + assert py.builtin.bytes == str + else: + assert py.builtin.text == str + assert py.builtin.bytes == bytes + def test_totext_badutf8(): # this was in printouts within the pytest testsuite # totext would fail https://bitbucket.org/hpk42/py/changeset/2ccc7c44763d/ changeset: 2ccc7c44763d user: hpk42 date: 2012-10-25 11:15:01 summary: fix pytest issue207 rewrite heuristic to find statements to use _ast module affected #: 6 files diff -r e6cf2e2fee5156e174a547beeb7d827d723a944b -r 2ccc7c44763d32e794d46e12933053c07962eb52 CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Changes between 1.4.10 and 1.4.11.dev ================================================== +- use _ast to determine statement ranges when printing tracebacks - + avoids multi-second delays on some large test modules - fix an internal test to not use class-denoted pytest_funcarg__ - fix a doc link to bug tracker - try to make terminal.write() printing more robust against diff -r e6cf2e2fee5156e174a547beeb7d827d723a944b -r 2ccc7c44763d32e794d46e12933053c07962eb52 py/__init__.py --- a/py/__init__.py +++ b/py/__init__.py @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = '1.4.11.dev4' +__version__ = '1.4.11.dev5' from py import _apipkg diff -r e6cf2e2fee5156e174a547beeb7d827d723a944b -r 2ccc7c44763d32e794d46e12933053c07962eb52 py/_code/code.py --- a/py/_code/code.py +++ b/py/_code/code.py @@ -166,20 +166,12 @@ if source is None: return None start = self.getfirstlinesource() - end = self.lineno try: - _, end = source.getstatementrange(end) - except (IndexError, ValueError): + _, end = source.getstatementrange(self.lineno) + except SyntaxError: end = self.lineno + 1 - # heuristic to stop displaying source on e.g. - # if something: # assume this causes a NameError - # # _this_ lines and the one - # below we don't want from entry.getsource() - for i in range(self.lineno, end): - if source[i].rstrip().endswith(':'): - end = i + 1 - break return source[start:end] + source = property(getsource) def ishidden(self): diff -r e6cf2e2fee5156e174a547beeb7d827d723a944b -r 2ccc7c44763d32e794d46e12933053c07962eb52 py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -108,47 +108,10 @@ def getstatementrange(self, lineno, assertion=False): """ return (start, end) tuple which spans the minimal statement region which containing the given lineno. - raise an IndexError if no such statementrange can be found. """ - # XXX there must be a better than these heuristic ways ... - # XXX there may even be better heuristics :-) if not (0 <= lineno < len(self)): raise IndexError("lineno out of range") - - # 1. find the start of the statement - from codeop import compile_command - for start in range(lineno, -1, -1): - if assertion: - line = self.lines[start] - # the following lines are not fully tested, change with care - if 'super' in line and 'self' in line and '__init__' in line: - raise IndexError("likely a subclass") - if "assert" not in line and "raise" not in line: - continue - trylines = self.lines[start:lineno+1] - # quick hack to prepare parsing an indented line with - # compile_command() (which errors on "return" outside defs) - trylines.insert(0, 'def xxx():') - trysource = '\n '.join(trylines) - # ^ space here - try: - compile_command(trysource) - except (SyntaxError, OverflowError, ValueError): - continue - - # 2. find the end of the statement - for end in range(lineno+1, len(self)+1): - trysource = self[start:end] - if trysource.isparseable(): - return start, end - raise IndexError("no valid source range around line %d " % (lineno,)) - - def getblockend(self, lineno): - # XXX - lines = [x + '\n' for x in self.lines[lineno:]] - blocklines = inspect.getblock(lines) - #print blocklines - return lineno + len(blocklines) - 1 + return getstatementrange_ast(lineno, self) def deindent(self, offset=None): """ return a new source object deindented by offset. @@ -352,3 +315,65 @@ # Add any lines we didn't see. E.g. if an exception was raised. newlines.extend(lines[len(newlines):]) return newlines + +def get_statement_startend(lineno, nodelist): + from bisect import bisect_right, bisect_left + # lineno starts at 0 + nextlineno = None + while 1: + lineno_list = [x.lineno-1 for x in nodelist] # ast indexes start at 1 + insert_index = bisect_right(lineno_list, lineno) + if insert_index >= len(nodelist): + insert_index -= 1 + elif lineno < (nodelist[insert_index].lineno - 1) and insert_index > 0: + insert_index -= 1 + assert lineno >= (nodelist[insert_index].lineno - 1) + nextnode = nodelist[insert_index] + + try: + nextlineno = nodelist[insert_index+1].lineno - 1 + except IndexError: + pass + lastnodelist = nodelist + nodelist = getnodelist(nextnode) + if not nodelist: + start, end = nextnode.lineno-1, nextlineno + start = min(lineno, start) + assert start <= lineno and (end is None or lineno < end) + #print "returning", start, end + return start, end + +def getnodelist(node): + import _ast + l = [] + #print "node", node, "fields", node._fields, "lineno", getattr(node, "lineno", 0) - 1 + for subname in "test", "type", "body", "handlers", "orelse", "finalbody": + attr = getattr(node, subname, None) + if attr is not None: + if isinstance(attr, list): + l.extend(attr) + elif hasattr(attr, "lineno"): + l.append(attr) + #print "returning nodelist", l + return l + +def getstatementrange_ast(lineno, source): + content = "\n".join(source.lines) + #print "compiling lines", len(source.lines) + astnode = compile(content, "source", "exec", 1024) # 1024 for AST + start, end = get_statement_startend(lineno, getnodelist(astnode)) + # we need to correct the end: + # - ast-parsing strips comments + # - else statements do not have a separate lineno + # - there might be empty lines + if end is None: + end = len(source.lines) + while end: + line = source.lines[end-1].lstrip() + if (not line or line.startswith("#") or line.startswith("else:") or + line.startswith("finally:")): + end -= 1 + else: + break + return start, end + diff -r e6cf2e2fee5156e174a547beeb7d827d723a944b -r 2ccc7c44763d32e794d46e12933053c07962eb52 setup.py --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ name='py', description='library with cross-python path, ini-parsing, io, code, log facilities', long_description = open('README.txt').read(), - version='1.4.11.dev4', + version='1.4.11.dev5', url='http://pylib.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff -r e6cf2e2fee5156e174a547beeb7d827d723a944b -r 2ccc7c44763d32e794d46e12933053c07962eb52 testing/code/test_source.py --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -178,10 +178,12 @@ def test_getstatementrange_triple_quoted(self): #print str(self.source) - source = Source("""''' - '''""") + source = Source("""hello(''' + ''')""") + s = source.getstatement(0) + assert s == str(source) s = source.getstatement(1) - assert eval(str(s)) + assert s == str(source) def test_getstatementrange_within_constructs(self): source = Source("""\ @@ -194,12 +196,13 @@ 42 """) assert len(source) == 7 - assert source.getstatementrange(0) == (0, 7) - assert source.getstatementrange(1) == (1, 5) + # check all lineno's that could occur in a traceback + #assert source.getstatementrange(0) == (0, 7) + #assert source.getstatementrange(1) == (1, 5) assert source.getstatementrange(2) == (2, 3) - assert source.getstatementrange(3) == (1, 5) + assert source.getstatementrange(3) == (3, 4) assert source.getstatementrange(4) == (4, 5) - assert source.getstatementrange(5) == (0, 7) + #assert source.getstatementrange(5) == (0, 7) assert source.getstatementrange(6) == (6, 7) def test_getstatementrange_bug(self): @@ -238,7 +241,7 @@ def test_getstatementrange_with_syntaxerror_issue7(self): source = Source(":") - py.test.raises(IndexError, lambda: source.getstatementrange(0)) + py.test.raises(SyntaxError, lambda: source.getstatementrange(0)) @py.test.mark.skipif("sys.version_info < (2,6)") def test_compile_to_ast(self): @@ -455,3 +458,123 @@ def __call__(self): pass py.test.raises(TypeError, lambda: py.code.Code(Hello)) + + +def getstatement(lineno, source): + from py._code.source import getstatementrange_ast + source = py.code.Source(source, deindent=False) + start, end = getstatementrange_ast(lineno, source) + return source[start:end] + +def test_oneline(): + source = getstatement(0, "raise ValueError") + assert str(source) == "raise ValueError" + +def test_oneline_and_comment(): + source = getstatement(0, "raise ValueError\n#hello") + assert str(source) == "raise ValueError" + +def XXXtest_multiline(): + source = getstatement(0, """\ +raise ValueError( + 23 +) +x = 3 +""") + assert str(source) == "raise ValueError(\n 23\n)" + +class TestTry: + source = """\ +try: + raise ValueError +except Something: + raise IndexError(1) +else: + raise KeyError() +""" + + def test_body(self): + source = getstatement(1, self.source) + assert str(source) == " raise ValueError" + + def test_except_line(self): + source = getstatement(2, self.source) + assert str(source) == "except Something:" + + def test_except_body(self): + source = getstatement(3, self.source) + assert str(source) == " raise IndexError(1)" + + def test_else(self): + source = getstatement(5, self.source) + assert str(source) == " raise KeyError()" + +class TestTryFinally: + source = """\ +try: + raise ValueError +finally: + raise IndexError(1) +""" + + def test_body(self): + source = getstatement(1, self.source) + assert str(source) == " raise ValueError" + + def test_finally(self): + source = getstatement(3, self.source) + assert str(source) == " raise IndexError(1)" + + + +class TestIf: + source = """\ +if 1: + y = 3 +elif False: + y = 5 +else: + y = 7 +""" + + def test_body(self): + source = getstatement(1, self.source) + assert str(source) == " y = 3" + + def test_elif_clause(self): + source = getstatement(2, self.source) + assert str(source) == "elif False:" + + def test_elif(self): + source = getstatement(3, self.source) + assert str(source) == " y = 5" + + def test_else(self): + source = getstatement(5, self.source) + assert str(source) == " y = 7" + +def test_semicolon(): + s = """\ +hello ; pytest.skip() +""" + source = getstatement(0, s) + assert str(source) == s.strip() + +def test_def_online(): + s = """\ +def func(): raise ValueError(42) + +def something(): + pass +""" + source = getstatement(0, s) + assert str(source) == "def func(): raise ValueError(42)" + +def XXX_test_expression_multiline(): + source = """\ +something +''' +'''""" + result = getstatement(1, source) + assert str(result) == "'''\n'''" + https://bitbucket.org/hpk42/py/changeset/2481690036ba/ changeset: 2481690036ba user: hpk42 date: 2012-10-25 11:38:25 summary: introduce an optional ASTcache passed in by exception-info printing to avoid reparsing the same module over and over affected #: 5 files diff -r 2ccc7c44763d32e794d46e12933053c07962eb52 -r 2481690036bae55d4cb613cba41490ed3839105e py/__init__.py --- a/py/__init__.py +++ b/py/__init__.py @@ -8,7 +8,7 @@ (c) Holger Krekel and others, 2004-2010 """ -__version__ = '1.4.11.dev5' +__version__ = '1.4.11.dev6' from py import _apipkg diff -r 2ccc7c44763d32e794d46e12933053c07962eb52 -r 2481690036bae55d4cb613cba41490ed3839105e py/_code/code.py --- a/py/_code/code.py +++ b/py/_code/code.py @@ -160,16 +160,28 @@ # on Jython this firstlineno can be -1 apparently return max(self.frame.code.firstlineno, 0) - def getsource(self): + def getsource(self, astcache=None): """ return failing source code. """ + # we use the passed in astcache to not reparse asttrees + # within exception info printing + from py._code.source import getstatementrange_ast source = self.frame.code.fullsource if source is None: return None + key = astnode = None + if astcache is not None: + key = self.frame.code.path + if key is not None: + astnode = astcache.get(key, None) start = self.getfirstlinesource() try: - _, end = source.getstatementrange(self.lineno) + astnode, _, end = getstatementrange_ast(self.lineno, source, + astnode=astnode) except SyntaxError: end = self.lineno + 1 + else: + if key is not None: + astcache[key] = astnode return source[start:end] source = property(getsource) @@ -391,6 +403,7 @@ self.tbfilter = tbfilter self.funcargs = funcargs self.abspath = abspath + self.astcache = {} def _getindent(self, source): # figure out indent for given source @@ -408,7 +421,7 @@ return 4 + (len(s) - len(s.lstrip())) def _getentrysource(self, entry): - source = entry.getsource() + source = entry.getsource(self.astcache) if source is not None: source = source.deindent() return source diff -r 2ccc7c44763d32e794d46e12933053c07962eb52 -r 2481690036bae55d4cb613cba41490ed3839105e py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -111,7 +111,8 @@ """ if not (0 <= lineno < len(self)): raise IndexError("lineno out of range") - return getstatementrange_ast(lineno, self) + ast, start, end = getstatementrange_ast(lineno, self) + return start, end def deindent(self, offset=None): """ return a new source object deindented by offset. @@ -357,10 +358,10 @@ #print "returning nodelist", l return l -def getstatementrange_ast(lineno, source): - content = "\n".join(source.lines) - #print "compiling lines", len(source.lines) - astnode = compile(content, "source", "exec", 1024) # 1024 for AST +def getstatementrange_ast(lineno, source, astnode=None): + if astnode is None: + content = "\n".join(source.lines) + astnode = compile(content, "source", "exec", 1024) # 1024 for AST start, end = get_statement_startend(lineno, getnodelist(astnode)) # we need to correct the end: # - ast-parsing strips comments @@ -375,5 +376,5 @@ end -= 1 else: break - return start, end + return astnode, start, end diff -r 2ccc7c44763d32e794d46e12933053c07962eb52 -r 2481690036bae55d4cb613cba41490ed3839105e setup.py --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ name='py', description='library with cross-python path, ini-parsing, io, code, log facilities', long_description = open('README.txt').read(), - version='1.4.11.dev5', + version='1.4.11.dev6', url='http://pylib.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff -r 2ccc7c44763d32e794d46e12933053c07962eb52 -r 2481690036bae55d4cb613cba41490ed3839105e testing/code/test_source.py --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -463,7 +463,7 @@ def getstatement(lineno, source): from py._code.source import getstatementrange_ast source = py.code.Source(source, deindent=False) - start, end = getstatementrange_ast(lineno, source) + ast, start, end = getstatementrange_ast(lineno, source) return source[start:end] def test_oneline(): Repository URL: https://bitbucket.org/hpk42/py/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email. _______________________________________________ py-svn mailing list py-svn@codespeak.net http://codespeak.net/mailman/listinfo/py-svn
