This seems to have caused the Windows buildbots to fail.
On Fri, Jan 11, 2013 at 5:40 AM, serhiy.storchaka <python-check...@python.org> wrote: > http://hg.python.org/cpython/rev/8452c23139c6 > changeset: 81407:8452c23139c6 > parent: 81399:5ec8daab477a > parent: 81406:01df1f7841b2 > user: Serhiy Storchaka <storch...@gmail.com> > date: Fri Jan 11 12:12:32 2013 +0200 > summary: > Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. > Now pindent.py works with a "with" statement. pindent.py no longer produces > improper indentation. pindent.py now works with continued lines broken after > "class" or "def" keywords and with continuations at the start of line. Added > regression tests for pindent.py. Modernized pindent.py. > > files: > Lib/test/test_tools.py | 326 ++++++++++++++++++++++++++- > Misc/NEWS | 7 + > Tools/scripts/pindent.py | 166 +++++-------- > 3 files changed, 393 insertions(+), 106 deletions(-) > > > diff --git a/Lib/test/test_tools.py b/Lib/test/test_tools.py > --- a/Lib/test/test_tools.py > +++ b/Lib/test/test_tools.py > @@ -9,10 +9,13 @@ > import importlib.machinery > import unittest > from unittest import mock > +import shutil > +import subprocess > import sysconfig > import tempfile > +import textwrap > from test import support > -from test.script_helper import assert_python_ok > +from test.script_helper import assert_python_ok, temp_dir > > if not sysconfig.is_python_build(): > # XXX some installers do contain the tools, should we detect that > @@ -36,6 +39,327 @@ > self.assertGreater(err, b'') > > > +class PindentTests(unittest.TestCase): > + script = os.path.join(scriptsdir, 'pindent.py') > + > + def assertFileEqual(self, fn1, fn2): > + with open(fn1) as f1, open(fn2) as f2: > + self.assertEqual(f1.readlines(), f2.readlines()) > + > + def pindent(self, source, *args): > + with subprocess.Popen( > + (sys.executable, self.script) + args, > + stdin=subprocess.PIPE, stdout=subprocess.PIPE, > + universal_newlines=True) as proc: > + out, err = proc.communicate(source) > + self.assertIsNone(err) > + return out > + > + def lstriplines(self, data): > + return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n' > + > + def test_selftest(self): > + with temp_dir() as directory: > + data_path = os.path.join(directory, '_test.py') > + with open(self.script) as f: > + closed = f.read() > + with open(data_path, 'w') as f: > + f.write(closed) > + > + rc, out, err = assert_python_ok(self.script, '-d', data_path) > + self.assertEqual(out, b'') > + self.assertEqual(err, b'') > + backup = data_path + '~' > + self.assertTrue(os.path.exists(backup)) > + with open(backup) as f: > + self.assertEqual(f.read(), closed) > + with open(data_path) as f: > + clean = f.read() > + compile(clean, '_test.py', 'exec') > + self.assertEqual(self.pindent(clean, '-c'), closed) > + self.assertEqual(self.pindent(closed, '-d'), clean) > + > + rc, out, err = assert_python_ok(self.script, '-c', data_path) > + self.assertEqual(out, b'') > + self.assertEqual(err, b'') > + with open(backup) as f: > + self.assertEqual(f.read(), clean) > + with open(data_path) as f: > + self.assertEqual(f.read(), closed) > + > + broken = self.lstriplines(closed) > + with open(data_path, 'w') as f: > + f.write(broken) > + rc, out, err = assert_python_ok(self.script, '-r', data_path) > + self.assertEqual(out, b'') > + self.assertEqual(err, b'') > + with open(backup) as f: > + self.assertEqual(f.read(), broken) > + with open(data_path) as f: > + indented = f.read() > + compile(indented, '_test.py', 'exec') > + self.assertEqual(self.pindent(broken, '-r'), indented) > + > + def pindent_test(self, clean, closed): > + self.assertEqual(self.pindent(clean, '-c'), closed) > + self.assertEqual(self.pindent(closed, '-d'), clean) > + broken = self.lstriplines(closed) > + self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed) > + > + def test_statements(self): > + clean = textwrap.dedent("""\ > + if a: > + pass > + > + if a: > + pass > + else: > + pass > + > + if a: > + pass > + elif: > + pass > + else: > + pass > + > + while a: > + break > + > + while a: > + break > + else: > + pass > + > + for i in a: > + break > + > + for i in a: > + break > + else: > + pass > + > + try: > + pass > + finally: > + pass > + > + try: > + pass > + except TypeError: > + pass > + except ValueError: > + pass > + else: > + pass > + > + try: > + pass > + except TypeError: > + pass > + except ValueError: > + pass > + finally: > + pass > + > + with a: > + pass > + > + class A: > + pass > + > + def f(): > + pass > + """) > + > + closed = textwrap.dedent("""\ > + if a: > + pass > + # end if > + > + if a: > + pass > + else: > + pass > + # end if > + > + if a: > + pass > + elif: > + pass > + else: > + pass > + # end if > + > + while a: > + break > + # end while > + > + while a: > + break > + else: > + pass > + # end while > + > + for i in a: > + break > + # end for > + > + for i in a: > + break > + else: > + pass > + # end for > + > + try: > + pass > + finally: > + pass > + # end try > + > + try: > + pass > + except TypeError: > + pass > + except ValueError: > + pass > + else: > + pass > + # end try > + > + try: > + pass > + except TypeError: > + pass > + except ValueError: > + pass > + finally: > + pass > + # end try > + > + with a: > + pass > + # end with > + > + class A: > + pass > + # end class A > + > + def f(): > + pass > + # end def f > + """) > + self.pindent_test(clean, closed) > + > + def test_multilevel(self): > + clean = textwrap.dedent("""\ > + def foobar(a, b): > + if a == b: > + a = a+1 > + elif a < b: > + b = b-1 > + if b > a: a = a-1 > + else: > + print 'oops!' > + """) > + closed = textwrap.dedent("""\ > + def foobar(a, b): > + if a == b: > + a = a+1 > + elif a < b: > + b = b-1 > + if b > a: a = a-1 > + # end if > + else: > + print 'oops!' > + # end if > + # end def foobar > + """) > + self.pindent_test(clean, closed) > + > + def test_preserve_indents(self): > + clean = textwrap.dedent("""\ > + if a: > + if b: > + pass > + """) > + closed = textwrap.dedent("""\ > + if a: > + if b: > + pass > + # end if > + # end if > + """) > + self.assertEqual(self.pindent(clean, '-c'), closed) > + self.assertEqual(self.pindent(closed, '-d'), clean) > + broken = self.lstriplines(closed) > + self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed) > + clean = textwrap.dedent("""\ > + if a: > + \tif b: > + \t\tpass > + """) > + closed = textwrap.dedent("""\ > + if a: > + \tif b: > + \t\tpass > + \t# end if > + # end if > + """) > + self.assertEqual(self.pindent(clean, '-c'), closed) > + self.assertEqual(self.pindent(closed, '-d'), clean) > + broken = self.lstriplines(closed) > + self.assertEqual(self.pindent(broken, '-r'), closed) > + > + def test_escaped_newline(self): > + clean = textwrap.dedent("""\ > + class\\ > + \\ > + A: > + def\ > + \\ > + f: > + pass > + """) > + closed = textwrap.dedent("""\ > + class\\ > + \\ > + A: > + def\ > + \\ > + f: > + pass > + # end def f > + # end class A > + """) > + self.assertEqual(self.pindent(clean, '-c'), closed) > + self.assertEqual(self.pindent(closed, '-d'), clean) > + > + def test_empty_line(self): > + clean = textwrap.dedent("""\ > + if a: > + > + pass > + """) > + closed = textwrap.dedent("""\ > + if a: > + > + pass > + # end if > + """) > + self.pindent_test(clean, closed) > + > + def test_oneline(self): > + clean = textwrap.dedent("""\ > + if a: pass > + """) > + closed = textwrap.dedent("""\ > + if a: pass > + # end if > + """) > + self.pindent_test(clean, closed) > + > + > class TestSundryScripts(unittest.TestCase): > # At least make sure the rest don't have syntax errors. When tests are > # added for a script it should be added to the whitelist below. > diff --git a/Misc/NEWS b/Misc/NEWS > --- a/Misc/NEWS > +++ b/Misc/NEWS > @@ -621,6 +621,8 @@ > Tests > ----- > > +- Issue #15539: Added regression tests for Tools/scripts/pindent.py. > + > - Issue #16836: Enable IPv6 support even if IPv6 is disabled on the build > host. > > - Issue #16925: test_configparser now works with unittest test discovery. > @@ -777,6 +779,11 @@ > Tools/Demos > ----------- > > +- Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now > + pindent.py works with a "with" statement. pindent.py no longer produces > + improper indentation. pindent.py now works with continued lines broken > after > + "class" or "def" keywords and with continuations at the start of line. > + > - Issue #11797: Add a 2to3 fixer that maps reload() to imp.reload(). > > - Issue #10966: Remove the concept of unexpected skipped tests. > diff --git a/Tools/scripts/pindent.py b/Tools/scripts/pindent.py > --- a/Tools/scripts/pindent.py > +++ b/Tools/scripts/pindent.py > @@ -79,8 +79,9 @@ > # Defaults > STEPSIZE = 8 > TABSIZE = 8 > -EXPANDTABS = 0 > +EXPANDTABS = False > > +import io > import re > import sys > > @@ -89,7 +90,8 @@ > next['while'] = next['for'] = 'else', 'end' > next['try'] = 'except', 'finally' > next['except'] = 'except', 'else', 'finally', 'end' > -next['else'] = next['finally'] = next['def'] = next['class'] = 'end' > +next['else'] = next['finally'] = next['with'] = \ > + next['def'] = next['class'] = 'end' > next['end'] = () > start = 'if', 'while', 'for', 'try', 'with', 'def', 'class' > > @@ -105,11 +107,11 @@ > self.expandtabs = expandtabs > self._write = fpo.write > self.kwprog = re.compile( > - r'^\s*(?P<kw>[a-z]+)' > - r'(\s+(?P<id>[a-zA-Z_]\w*))?' > + r'^(?:\s|\\\n)*(?P<kw>[a-z]+)' > + r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?' > r'[^\w]') > self.endprog = re.compile( > - r'^\s*#?\s*end\s+(?P<kw>[a-z]+)' > + r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)' > r'(\s+(?P<id>[a-zA-Z_]\w*))?' > r'[^\w]') > self.wsprog = re.compile(r'^[ \t]*') > @@ -125,7 +127,7 @@ > > def readline(self): > line = self.fpi.readline() > - if line: self.lineno = self.lineno + 1 > + if line: self.lineno += 1 > # end if > return line > # end def readline > @@ -143,27 +145,24 @@ > line2 = self.readline() > if not line2: break > # end if > - line = line + line2 > + line += line2 > # end while > return line > # end def getline > > - def putline(self, line, indent = None): > - if indent is None: > - self.write(line) > - return > + def putline(self, line, indent): > + tabs, spaces = divmod(indent*self.indentsize, self.tabsize) > + i = self.wsprog.match(line).end() > + line = line[i:] > + if line[:1] not in ('\n', '\r', ''): > + line = '\t'*tabs + ' '*spaces + line > # end if > - tabs, spaces = divmod(indent*self.indentsize, self.tabsize) > - i = 0 > - m = self.wsprog.match(line) > - if m: i = m.end() > - # end if > - self.write('\t'*tabs + ' '*spaces + line[i:]) > + self.write(line) > # end def putline > > def reformat(self): > stack = [] > - while 1: > + while True: > line = self.getline() > if not line: break # EOF > # end if > @@ -173,10 +172,9 @@ > kw2 = m.group('kw') > if not stack: > self.error('unexpected end') > - elif stack[-1][0] != kw2: > + elif stack.pop()[0] != kw2: > self.error('unmatched end') > # end if > - del stack[-1:] > self.putline(line, len(stack)) > continue > # end if > @@ -208,23 +206,23 @@ > def delete(self): > begin_counter = 0 > end_counter = 0 > - while 1: > + while True: > line = self.getline() > if not line: break # EOF > # end if > m = self.endprog.match(line) > if m: > - end_counter = end_counter + 1 > + end_counter += 1 > continue > # end if > m = self.kwprog.match(line) > if m: > kw = m.group('kw') > if kw in start: > - begin_counter = begin_counter + 1 > + begin_counter += 1 > # end if > # end if > - self.putline(line) > + self.write(line) > # end while > if begin_counter - end_counter < 0: > sys.stderr.write('Warning: input contained more end tags than > expected\n') > @@ -234,17 +232,12 @@ > # end def delete > > def complete(self): > - self.indentsize = 1 > stack = [] > todo = [] > - thisid = '' > - current, firstkw, lastkw, topid = 0, '', '', '' > - while 1: > + currentws = thisid = firstkw = lastkw = topid = '' > + while True: > line = self.getline() > - i = 0 > - m = self.wsprog.match(line) > - if m: i = m.end() > - # end if > + i = self.wsprog.match(line).end() > m = self.endprog.match(line) > if m: > thiskw = 'end' > @@ -269,7 +262,9 @@ > thiskw = '' > # end if > # end if > - indent = len(line[:i].expandtabs(self.tabsize)) > + indentws = line[:i] > + indent = len(indentws.expandtabs(self.tabsize)) > + current = len(currentws.expandtabs(self.tabsize)) > while indent < current: > if firstkw: > if topid: > @@ -278,11 +273,11 @@ > else: > s = '# end %s\n' % firstkw > # end if > - self.putline(s, current) > + self.write(currentws + s) > firstkw = lastkw = '' > # end if > - current, firstkw, lastkw, topid = stack[-1] > - del stack[-1] > + currentws, firstkw, lastkw, topid = stack.pop() > + current = len(currentws.expandtabs(self.tabsize)) > # end while > if indent == current and firstkw: > if thiskw == 'end': > @@ -297,18 +292,18 @@ > else: > s = '# end %s\n' % firstkw > # end if > - self.putline(s, current) > + self.write(currentws + s) > firstkw = lastkw = topid = '' > # end if > # end if > if indent > current: > - stack.append((current, firstkw, lastkw, topid)) > + stack.append((currentws, firstkw, lastkw, topid)) > if thiskw and thiskw not in start: > # error > thiskw = '' > # end if > - current, firstkw, lastkw, topid = \ > - indent, thiskw, thiskw, thisid > + currentws, firstkw, lastkw, topid = \ > + indentws, thiskw, thiskw, thisid > # end if > if thiskw: > if thiskw in start: > @@ -326,7 +321,6 @@ > self.write(line) > # end while > # end def complete > - > # end class PythonIndenter > > # Simplified user interface > @@ -352,76 +346,34 @@ > pi.reformat() > # end def reformat_filter > > -class StringReader: > - def __init__(self, buf): > - self.buf = buf > - self.pos = 0 > - self.len = len(self.buf) > - # end def __init__ > - def read(self, n = 0): > - if n <= 0: > - n = self.len - self.pos > - else: > - n = min(n, self.len - self.pos) > - # end if > - r = self.buf[self.pos : self.pos + n] > - self.pos = self.pos + n > - return r > - # end def read > - def readline(self): > - i = self.buf.find('\n', self.pos) > - return self.read(i + 1 - self.pos) > - # end def readline > - def readlines(self): > - lines = [] > - line = self.readline() > - while line: > - lines.append(line) > - line = self.readline() > - # end while > - return lines > - # end def readlines > - # seek/tell etc. are left as an exercise for the reader > -# end class StringReader > - > -class StringWriter: > - def __init__(self): > - self.buf = '' > - # end def __init__ > - def write(self, s): > - self.buf = self.buf + s > - # end def write > - def getvalue(self): > - return self.buf > - # end def getvalue > -# end class StringWriter > - > def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, > expandtabs = EXPANDTABS): > - input = StringReader(source) > - output = StringWriter() > + input = io.StringIO(source) > + output = io.StringIO() > pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) > pi.complete() > return output.getvalue() > # end def complete_string > > def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs > = EXPANDTABS): > - input = StringReader(source) > - output = StringWriter() > + input = io.StringIO(source) > + output = io.StringIO() > pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) > pi.delete() > return output.getvalue() > # end def delete_string > > def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, > expandtabs = EXPANDTABS): > - input = StringReader(source) > - output = StringWriter() > + input = io.StringIO(source) > + output = io.StringIO() > pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) > pi.reformat() > return output.getvalue() > # end def reformat_string > > def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, > expandtabs = EXPANDTABS): > - source = open(filename, 'r').read() > + with open(filename, 'r') as f: > + source = f.read() > + # end with > result = complete_string(source, stepsize, tabsize, expandtabs) > if source == result: return 0 > # end if > @@ -429,14 +381,16 @@ > try: os.rename(filename, filename + '~') > except OSError: pass > # end try > - f = open(filename, 'w') > - f.write(result) > - f.close() > + with open(filename, 'w') as f: > + f.write(result) > + # end with > return 1 > # end def complete_file > > def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs > = EXPANDTABS): > - source = open(filename, 'r').read() > + with open(filename, 'r') as f: > + source = f.read() > + # end with > result = delete_string(source, stepsize, tabsize, expandtabs) > if source == result: return 0 > # end if > @@ -444,14 +398,16 @@ > try: os.rename(filename, filename + '~') > except OSError: pass > # end try > - f = open(filename, 'w') > - f.write(result) > - f.close() > + with open(filename, 'w') as f: > + f.write(result) > + # end with > return 1 > # end def delete_file > > def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, > expandtabs = EXPANDTABS): > - source = open(filename, 'r').read() > + with open(filename, 'r') as f: > + source = f.read() > + # end with > result = reformat_string(source, stepsize, tabsize, expandtabs) > if source == result: return 0 > # end if > @@ -459,9 +415,9 @@ > try: os.rename(filename, filename + '~') > except OSError: pass > # end try > - f = open(filename, 'w') > - f.write(result) > - f.close() > + with open(filename, 'w') as f: > + f.write(result) > + # end with > return 1 > # end def reformat_file > > @@ -474,7 +430,7 @@ > -r : reformat a completed program (use #end directives) > -s stepsize: indentation step (default %(STEPSIZE)d) > -t tabsize : the worth in spaces of a tab (default %(TABSIZE)d) > --e : expand TABs into spaces (defailt OFF) > +-e : expand TABs into spaces (default OFF) > [file] ... : files are changed in place, with backups in file~ > If no files are specified or a single - is given, > the program acts as a filter (reads stdin, writes stdout). > @@ -517,7 +473,7 @@ > elif o == '-t': > tabsize = int(a) > elif o == '-e': > - expandtabs = 1 > + expandtabs = True > # end if > # end for > if not action: > > -- > Repository URL: http://hg.python.org/cpython > > _______________________________________________ > Python-checkins mailing list > python-check...@python.org > http://mail.python.org/mailman/listinfo/python-checkins > _______________________________________________ Python-Dev mailing list Python-Dev@python.org http://mail.python.org/mailman/listinfo/python-dev Unsubscribe: http://mail.python.org/mailman/options/python-dev/archive%40mail-archive.com
