Author: Ronan Lamy <[email protected]> Branch: py3.7 Changeset: r97703:11316d0ce627 Date: 2019-10-01 21:32 +0100 http://bitbucket.org/pypy/pypy/changeset/11316d0ce627/
Log: hg merge py3.6 diff too long, truncating to 2000 out of 21056 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -40,11 +40,11 @@ Armin Rigo Maciej Fijalkowski Carl Friedrich Bolz-Tereick + Matti Picus Antonio Cuni Amaury Forgeot d'Arc - Matti Picus + Ronan Lamy Samuele Pedroni - Ronan Lamy Alex Gaynor Philip Jenvey Richard Plangger @@ -94,6 +94,7 @@ Jason Creighton Mark Young Alex Martelli + Andrew Lawrence Spenser Bauman Michal Bendowski Jan de Mooij @@ -106,11 +107,11 @@ Stefan Schwarzer Tomek Meka Valentino Volonghi + Stefan Beyer Patrick Maupin Devin Jeanpierre Bob Ippolito Bruno Gola - Andrew Lawrence David Malcolm Squeaky Edd Barrett @@ -124,7 +125,6 @@ Wenzhu Man Konstantin Lopuhin John Witulski - Stefan Beyer Jeremy Thurgood Greg Price Ivan Sichmann Freitas @@ -138,6 +138,7 @@ Pavel Vinogradov William Leslie Paweł Piotr Przeradowski + Stian Andreassen marky1991 Ilya Osadchiy Tobias Oberstein @@ -146,14 +147,13 @@ Taavi Burns Adrian Kuhn tav - Stian Andreassen Georg Brandl Joannah Nanjekye + Julian Berman Bert Freudenberg Wanja Saatkamp Mike Blume Gerald Klix - Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -204,6 +204,7 @@ Andrews Medina Aaron Iles Toby Watson + Lin Cheng Daniel Patrick Stuart Williams Antoine Pitrou @@ -245,6 +246,7 @@ Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Anthony Sottile Marc Abramowitz Arjun Naik Aaron Gallagher @@ -254,7 +256,6 @@ Omer Katz Jacek Generowicz Tomasz Dziopa - Lin Cheng Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -285,7 +286,6 @@ Lene Wagner Tomo Cocoa Miro Hrončok - Anthony Sottile David Lievens Neil Blakey-Milner Henrik Vendelbo @@ -294,11 +294,14 @@ Christoph Gerum Miguel de Val Borro Artur Lisiecki + [email protected] afteryu Toni Mattis + Vincent Michel Laurens Van Houtven Bobby Impollonia Roberto De Ioris + Yannick Jadoul Jeong YunWon Christopher Armstrong Aaron Tubbs @@ -312,6 +315,7 @@ Fabio Niephaus Akira Li Gustavo Niemeyer + [email protected] Nate Bragg Lucas Stadler roberto@goyle @@ -331,8 +335,12 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle + Paul Ganssle + Michal Kuffa Stephan Busemann + Bystroushaak Dan Colish + Ram Rachum timo Volodymyr Vladymyrov Daniel Neuhäuser @@ -342,18 +350,22 @@ Chris Lambacher John Aldis [email protected] + Yasen Kiprov Mike Bayer Rodrigo Araújo Daniil Yarancev Min RK OlivierBlanvillain + [email protected] Jonas Pfannschmidt Zearin Johan Forsberg Andrey Churin Dan Crosta [email protected] + Ryan Hileman Stanisław Halik + DeVerne Jones Julien Phalip Roman Podoliaka Steve Papanik @@ -369,17 +381,20 @@ Jim Hunziker shoma hosaka Buck Golemon + whitequark Iraklis D. JohnDoe yrttyr Michael Chermside Anna Ravencroft remarkablerocket + Ivan Petre Vijiac Berker Peksag Christian Muirhead soareschen Matthew Miller + Jesdi Konrad Delong Dinu Gherman pizi @@ -398,13 +413,16 @@ Markus Unterwaditzer Kristoffer Kleine Graham Markall + paugier Dan Loewenherz werat Filip Salomonsson Niclas Olofsson + Zsolt Cserna Chris Pressey Tobias Diaz Paul Graydon + mkuffa Nikolaos-Digenis Karagiannis Kurt Griffiths Ben Mather diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py --- a/extra_tests/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -66,7 +66,7 @@ int add43(int, ...); int globalvar42; const int globalconst42; - const char *const globalconsthello = "hello"; + const char *const globalconsthello; int no_such_function(int); int no_such_globalvar; struct foo_s; diff --git a/extra_tests/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_version.py rename from extra_tests/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_version.py --- a/extra_tests/cffi_tests/test_egg_version.py +++ b/extra_tests/cffi_tests/test_version.py @@ -1,6 +1,7 @@ from email.parser import Parser import py +from urllib.request import urlopen import cffi import pypy @@ -10,3 +11,11 @@ def test_egg_version(): info = Parser().parsestr(egg_info.read()) assert info['version'] == cffi.__version__ + +def test_pycparser_version(): + url = 'https://raw.githubusercontent.com/eliben/pycparser/master/pycparser/__init__.py' + source = urlopen(url).read().decode('utf8') + dest = py.path.local(__file__).join('..', '..', '..', 'lib_pypy', 'cffi', + '_pycparser', '__init__.py').read() + # if this fails, the vendored pycparser is not the latest version + assert source.strip() == dest.strip() diff --git a/extra_tests/test_semlock.py b/extra_tests/test_semlock.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_semlock.py @@ -0,0 +1,36 @@ +from _multiprocessing import SemLock +from threading import Thread +import _thread +import time + + +def test_notify_all(): + """A low-level variation on test_notify_all() in lib-python's + _test_multiprocessing.py + """ + N_THREADS = 1000 + lock = SemLock(0, 1, 1, "/test_notify_all", True) + results = [] + + def f(n): + if lock.acquire(timeout=5.): + results.append(n) + lock.release() + + threads = [Thread(target=f, args=(i,)) for i in range(N_THREADS)] + n_started = N_THREADS + with lock: + for t in threads: + try: + t.start() + except _thread.error: + # too many threads for this system + t.started = False + n_started -= 1 + else: + t.started = True + time.sleep(0.1) + for t in threads: + if t.started: + t.join() + assert len(results) == n_started diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py --- a/lib-python/3/stat.py +++ b/lib-python/3/stat.py @@ -40,6 +40,10 @@ S_IFIFO = 0o010000 # fifo (named pipe) S_IFLNK = 0o120000 # symbolic link S_IFSOCK = 0o140000 # socket file +# Fallbacks for uncommon platform-specific constants +S_IFDOOR = 0 +S_IFPORT = 0 +S_IFWHT = 0 # Functions to test for each file type @@ -71,6 +75,18 @@ """Return True if mode is from a socket.""" return S_IFMT(mode) == S_IFSOCK +def S_ISDOOR(mode): + """Return True if mode is from a door.""" + return False + +def S_ISPORT(mode): + """Return True if mode is from an event port.""" + return False + +def S_ISWHT(mode): + """Return True if mode is from a whiteout.""" + return False + # Names for permission bits S_ISUID = 0o4000 # set UID bit diff --git a/lib-python/3/test/test_concurrent_futures.py b/lib-python/3/test/test_concurrent_futures.py --- a/lib-python/3/test/test_concurrent_futures.py +++ b/lib-python/3/test/test_concurrent_futures.py @@ -636,6 +636,7 @@ futures_list.remove(future) wr = weakref.ref(future) del future + test.support.gc_collect() self.assertIsNone(wr()) futures_list[0].set_result("test") @@ -643,6 +644,7 @@ futures_list.remove(future) wr = weakref.ref(future) del future + test.support.gc_collect() self.assertIsNone(wr()) if futures_list: futures_list[0].set_result("test") @@ -742,6 +744,7 @@ for obj in self.executor.map(make_dummy_object, range(10)): wr = weakref.ref(obj) del obj + test.support.gc_collect() self.assertIsNone(wr()) diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py --- a/lib-python/3/test/test_dis.py +++ b/lib-python/3/test/test_dis.py @@ -145,7 +145,7 @@ 1) pass -_bug1333982 = """\ +dis_bug1333982 = """\ %3d 0 LOAD_CONST 1 (0) 2 POP_JUMP_IF_TRUE 26 4 LOAD_GLOBAL 0 (AssertionError) diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py --- a/lib-python/3/test/test_peepholer.py +++ b/lib-python/3/test/test_peepholer.py @@ -186,9 +186,11 @@ code = compile('a=1<<1000', '', 'single') self.assertInBytecode(code, 'LOAD_CONST', 1000) self.assertNotIn(1<<1000, code.co_consts) - code = compile('a=2**1000', '', 'single') - self.assertInBytecode(code, 'LOAD_CONST', 1000) - self.assertNotIn(2**1000, code.co_consts) + # difference to CPython: PyPy allows slightly larger constants to be + # created + code = compile('a=2**10000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 10000) + self.assertNotIn(2**10000, code.co_consts) @cpython_only # we currently not bother to implement that def test_binary_subscr_on_unicode(self): diff --git a/lib-python/3/test/test_readline.py b/lib-python/3/test/test_readline.py --- a/lib-python/3/test/test_readline.py +++ b/lib-python/3/test/test_readline.py @@ -243,7 +243,7 @@ # See https://cnswww.cns.cwru.edu/php/chet/readline/CHANGES # - editline: history size is broken on OS X 10.11.6. # Newer versions were not tested yet. - @unittest.skipIf(readline._READLINE_VERSION < 0x600, + @unittest.skipIf(getattr(readline, "_READLINE_VERSION", 0x601) < 0x600, "this readline version does not support history-size") @unittest.skipIf(is_editline, "editline history size configuration is broken") diff --git a/lib-python/3/test/test_regrtest.py b/lib-python/3/test/test_regrtest.py --- a/lib-python/3/test/test_regrtest.py +++ b/lib-python/3/test/test_regrtest.py @@ -258,7 +258,6 @@ self.checkError([opt, '0', '-T'], "don't go together") self.checkError([opt, '0', '-T'], "don't go together") self.checkError([opt, '0', '-l'], "don't go together") - self.checkError([opt, '0', '-M', '4G'], "don't go together") def test_coverage(self): for opt in '-T', '--coverage': diff --git a/lib-python/3/test/test_threading.py b/lib-python/3/test/test_threading.py --- a/lib-python/3/test/test_threading.py +++ b/lib-python/3/test/test_threading.py @@ -561,6 +561,7 @@ self.assertEqual(err, b"") self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + @test.support.cpython_only @requires_type_collecting def test_main_thread_during_shutdown(self): # bpo-31516: current_thread() should still point to the main thread diff --git a/lib-python/3/test/test_time.py b/lib-python/3/test/test_time.py --- a/lib-python/3/test/test_time.py +++ b/lib-python/3/test/test_time.py @@ -588,6 +588,8 @@ self.skipTest('could not set locale.LC_ALL to fr_FR') # This should not cause an exception time.strftime("%B", (2009,2,1,0,0,0,0,0,0)) + # PyPy addition: + time.strftime("%B", (2009,8,1,0,0,0,0,0,0)).lower() class _TestAsctimeYear: diff --git a/lib-python/3/unittest/test/testmock/testhelpers.py b/lib-python/3/unittest/test/testmock/testhelpers.py --- a/lib-python/3/unittest/test/testmock/testhelpers.py +++ b/lib-python/3/unittest/test/testmock/testhelpers.py @@ -3,6 +3,8 @@ import types import unittest +from test.support import cpython_only + from unittest.mock import ( call, _Call, create_autospec, MagicMock, Mock, ANY, _CallList, patch, PropertyMock, _callable @@ -875,7 +877,7 @@ # plain data descriptor check_data_descriptor(foo.desc) - + @cpython_only # PyPy can easily extract a spec from a builtin function def test_autospec_on_bound_builtin_function(self): meth = types.MethodType(time.ctime, time.time()) self.assertIsInstance(meth(), str) diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py --- a/lib_pypy/_curses_build.py +++ b/lib_pypy/_curses_build.py @@ -34,6 +34,13 @@ #define NCURSES_OPAQUE 0 #endif + +/* ncurses 6 change behaviour and makes all pointers opaque, + lets define backward compatibility. It doesn't harm + previous versions */ + +#define NCURSES_INTERNALS 1 +#define NCURSES_REENTRANT 0 #include <ncurses.h> #include <panel.h> #include <term.h> diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py --- a/lib_pypy/_dbm.py +++ b/lib_pypy/_dbm.py @@ -149,7 +149,7 @@ lib = CDLL("/usr/lib/libdbm.dylib") # OS X _platform = 'osx' -library = "GNU gdbm" +library = "Berkeley DB" funcs = {} _init_func('open', (c_char_p, c_int, c_int), restype=c_void_p) diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -61,6 +61,8 @@ assert output_dir is not None from distutils.ccompiler import new_compiler + from distutils import log + log.set_verbosity(3) compiler = new_compiler() compiler.output_dir = output_dir @@ -72,7 +74,8 @@ ccflags = ['-fPIC', '-Wimplicit-function-declaration'] res = compiler.compile([os.path.join(thisdir, csource)], include_dirs=[include_dir], - extra_preargs=ccflags) + extra_preargs=ccflags, + ) object_filename = res[0] # set link options diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -746,6 +746,8 @@ self.__initialized = True def close(self): + if not self.__initialized: + raise ProgrammingError("Base Cursor.__init__ not called.") self.__connection._check_thread() self.__connection._check_closed() if self.__statement: @@ -1043,6 +1045,7 @@ return list(self) def __get_connection(self): + self.__check_cursor() return self.__connection connection = property(__get_connection) diff --git a/lib_pypy/_stat.py b/lib_pypy/_stat.py deleted file mode 100644 --- a/lib_pypy/_stat.py +++ /dev/null @@ -1,6 +0,0 @@ -# Assume not Solaris - -S_IFDOOR = 0 - -def S_ISDOOR(mode): - return False diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c --- a/lib_pypy/_testcapimodule.c +++ b/lib_pypy/_testcapimodule.c @@ -3132,6 +3132,8 @@ return PyLong_FromLong(r); } +#endif /* PYPY_VERSION */ + static int check_time_rounding(int round) { @@ -3192,6 +3194,8 @@ return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec); } +#ifndef PYPY_VERSION + static void slot_tp_del(PyObject *self) { @@ -4048,8 +4052,6 @@ Py_RETURN_NONE; } -#ifndef PYPY_VERSION - static PyObject * test_pytime_fromseconds(PyObject *self, PyObject *args) { @@ -4187,6 +4189,8 @@ return _PyTime_AsNanosecondsObject(ms); } +#ifndef PYPY_VERSION + static PyObject* get_recursion_depth(PyObject *self, PyObject *args) { @@ -4809,9 +4813,11 @@ {"crash_no_current_thread", (PyCFunction)crash_no_current_thread, METH_NOARGS}, #ifndef PYPY_VERSION {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, +#endif {"pytime_object_to_time_t", test_pytime_object_to_time_t, METH_VARARGS}, {"pytime_object_to_timeval", test_pytime_object_to_timeval, METH_VARARGS}, {"pytime_object_to_timespec", test_pytime_object_to_timespec, METH_VARARGS}, +#ifndef PYPY_VERSION {"with_tp_del", with_tp_del, METH_VARARGS}, #endif {"create_cfunction", create_cfunction, METH_NOARGS}, @@ -4877,7 +4883,6 @@ return_null_without_error, METH_NOARGS}, {"return_result_with_error", return_result_with_error, METH_NOARGS}, -#ifndef PYPY_VERSION {"PyTime_FromSeconds", test_pytime_fromseconds, METH_VARARGS}, {"PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS}, {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, @@ -4887,6 +4892,7 @@ #endif {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, +#ifndef PYPY_VERSION {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, @@ -4906,10 +4912,10 @@ {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, +#endif #ifdef W_STOPCODE {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif -#endif /* PYPY_VERSION */ {"get_mapping_keys", get_mapping_keys, METH_O}, {"get_mapping_values", get_mapping_values, METH_O}, {"get_mapping_items", get_mapping_items, METH_O}, diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py --- a/lib_pypy/_tkinter/tklib_build.py +++ b/lib_pypy/_tkinter/tklib_build.py @@ -36,8 +36,11 @@ for _ver in ['8.6', '8.5', '']: incdirs = [] linklibs = ['tcl' + _ver, 'tk' + _ver] - if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])): - found = True + for lib in ['/usr/lib/lib', '/usr/lib64/lib']: + if os.path.isfile(''.join([lib, linklibs[1], '.so'])): + found = True + break + if found: break if not found: sys.stderr.write("*** TCL libraries not found! Falling back...\n") diff --git a/lib_pypy/cffi/_pycparser/README b/lib_pypy/cffi/_pycparser/README --- a/lib_pypy/cffi/_pycparser/README +++ b/lib_pypy/cffi/_pycparser/README @@ -10,3 +10,8 @@ ^^^^^^^^^^^^^^^ yacctab='cffi._pycparser.yacctab', ^^^^^^^^^^^^^^^ + +Also, when updating the version of this in-place, you must regenerate the +lextab.py and yacctab.py files. They will be regenerated on import if they +are not found, so they should be removed, then regenrated, then the new +versions committed. diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py --- a/lib_pypy/cffi/_pycparser/__init__.py +++ b/lib_pypy/cffi/_pycparser/__init__.py @@ -4,12 +4,14 @@ # This package file exports some convenience functions for # interacting with pycparser # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #----------------------------------------------------------------- __all__ = ['c_lexer', 'c_parser', 'c_ast'] -__version__ = '2.14' +__version__ = '2.19' +import io +from subprocess import check_output from .c_parser import CParser @@ -27,7 +29,6 @@ When successful, returns the preprocessed file's contents. Errors from cpp will be printed out. """ - from subprocess import Popen, PIPE path_list = [cpp_path] if isinstance(cpp_args, list): path_list += cpp_args @@ -38,11 +39,7 @@ try: # Note the use of universal_newlines to treat all newlines # as \n for Python's purpose - # - pipe = Popen( path_list, - stdout=PIPE, - universal_newlines=True) - text = pipe.communicate()[0] + text = check_output(path_list, universal_newlines=True) except OSError as e: raise RuntimeError("Unable to invoke 'cpp'. " + 'Make sure its path was passed correctly\n' + @@ -85,7 +82,7 @@ if use_cpp: text = preprocess_file(filename, cpp_path, cpp_args) else: - with open(filename, 'rU') as f: + with io.open(filename) as f: text = f.read() if parser is None: diff --git a/lib_pypy/cffi/_pycparser/_ast_gen.py b/lib_pypy/cffi/_pycparser/_ast_gen.py --- a/lib_pypy/cffi/_pycparser/_ast_gen.py +++ b/lib_pypy/cffi/_pycparser/_ast_gen.py @@ -7,7 +7,7 @@ # The design of this module was inspired by astgen.py from the # Python 2.5 code-base. # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #----------------------------------------------------------------- import pprint @@ -63,6 +63,7 @@ contents: a list of contents - attributes and child nodes See comment at the top of the configuration file for details. """ + def __init__(self, name, contents): self.name = name self.all_entries = [] @@ -84,6 +85,8 @@ def generate_source(self): src = self._gen_init() src += '\n' + self._gen_children() + src += '\n' + self._gen_iter() + src += '\n' + self._gen_attr_names() return src @@ -131,6 +134,33 @@ return src + def _gen_iter(self): + src = ' def __iter__(self):\n' + + if self.all_entries: + for child in self.child: + src += ( + ' if self.%(child)s is not None:\n' + + ' yield self.%(child)s\n') % (dict(child=child)) + + for seq_child in self.seq_child: + src += ( + ' for child in (self.%(child)s or []):\n' + ' yield child\n') % (dict(child=seq_child)) + + if not (self.child or self.seq_child): + # Empty generator + src += ( + ' return\n' + + ' yield\n') + else: + # Empty generator + src += ( + ' return\n' + + ' yield\n') + + return src + def _gen_attr_names(self): src = " attr_names = (" + ''.join("%r, " % nm for nm in self.attr) + ')' return src @@ -150,7 +180,7 @@ # # AST Node classes. # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #----------------------------------------------------------------- @@ -159,11 +189,38 @@ _PROLOGUE_CODE = r''' import sys +def _repr(obj): + """ + Get the representation of an object, with dedicated pprint-like format for lists. + """ + if isinstance(obj, list): + return '[' + (',\n '.join((_repr(e).replace('\n', '\n ') for e in obj))) + '\n]' + else: + return repr(obj) class Node(object): __slots__ = () """ Abstract base class for AST nodes. """ + def __repr__(self): + """ Generates a python representation of the current node + """ + result = self.__class__.__name__ + '(' + + indent = '' + separator = '' + for name in self.__slots__[:-2]: + result += separator + result += indent + result += name + '=' + (_repr(getattr(self, name)).replace('\n', '\n ' + (' ' * (len(name) + len(self.__class__.__name__))))) + + separator = ',' + indent = '\n ' + (' ' * len(self.__class__.__name__)) + + result += indent + ')' + + return result + def children(self): """ A sequence of all children that are Nodes """ @@ -253,26 +310,29 @@ * Modeled after Python's own AST visiting facilities (the ast module of Python 3.0) """ + + _method_cache = None + def visit(self, node): """ Visit a node. """ - method = 'visit_' + node.__class__.__name__ - visitor = getattr(self, method, self.generic_visit) + + if self._method_cache is None: + self._method_cache = {} + + visitor = self._method_cache.get(node.__class__.__name__, None) + if visitor is None: + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + self._method_cache[node.__class__.__name__] = visitor + return visitor(node) def generic_visit(self, node): """ Called if no explicit visitor function exists for a node. Implements preorder visiting of the node. """ - for c_name, c in node.children(): + for c in node: self.visit(c) - ''' - - -if __name__ == "__main__": - import sys - ast_gen = ASTCodeGenerator('_c_ast.cfg') - ast_gen.generate(open('c_ast.py', 'w')) - diff --git a/lib_pypy/cffi/_pycparser/_build_tables.py b/lib_pypy/cffi/_pycparser/_build_tables.py --- a/lib_pypy/cffi/_pycparser/_build_tables.py +++ b/lib_pypy/cffi/_pycparser/_build_tables.py @@ -6,17 +6,21 @@ # Also generates AST code from the configuration file. # Should be called from the pycparser directory. # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #----------------------------------------------------------------- +# Insert '.' and '..' as first entries to the search path for modules. +# Restricted environments like embeddable python do not include the +# current working directory on startup. +import sys +sys.path[0:0] = ['.', '..'] + # Generate c_ast.py from _ast_gen import ASTCodeGenerator ast_gen = ASTCodeGenerator('_c_ast.cfg') ast_gen.generate(open('c_ast.py', 'w')) -import sys -sys.path[0:0] = ['.', '..'] from pycparser import c_parser # Generates the tables diff --git a/lib_pypy/cffi/_pycparser/_c_ast.cfg b/lib_pypy/cffi/_pycparser/_c_ast.cfg --- a/lib_pypy/cffi/_pycparser/_c_ast.cfg +++ b/lib_pypy/cffi/_pycparser/_c_ast.cfg @@ -9,7 +9,7 @@ # <name>** - a sequence of child nodes # <name> - an attribute # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #----------------------------------------------------------------- @@ -187,3 +187,5 @@ Union: [name, decls**] While: [cond*, stmt*] + +Pragma: [string] diff --git a/lib_pypy/cffi/_pycparser/ast_transforms.py b/lib_pypy/cffi/_pycparser/ast_transforms.py --- a/lib_pypy/cffi/_pycparser/ast_transforms.py +++ b/lib_pypy/cffi/_pycparser/ast_transforms.py @@ -3,7 +3,7 @@ # # Some utilities used by the parser to create a friendlier AST. # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #------------------------------------------------------------------------------ @@ -43,7 +43,7 @@ Default: break - The goal of this transform it to fix this mess, turning it into the + The goal of this transform is to fix this mess, turning it into the following: Switch @@ -74,7 +74,8 @@ # Goes over the children of the Compound below the Switch, adding them # either directly below new_compound or below the last Case as appropriate - for child in switch_node.stmt.block_items: + # (for `switch(cond) {}`, block_items would have been None) + for child in (switch_node.stmt.block_items or []): if isinstance(child, (c_ast.Case, c_ast.Default)): # If it's a Case/Default: # 1. Add it to the Compound and mark as "last case" diff --git a/lib_pypy/cffi/_pycparser/c_ast.py b/lib_pypy/cffi/_pycparser/c_ast.py --- a/lib_pypy/cffi/_pycparser/c_ast.py +++ b/lib_pypy/cffi/_pycparser/c_ast.py @@ -11,18 +11,45 @@ # # AST Node classes. # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #----------------------------------------------------------------- import sys +def _repr(obj): + """ + Get the representation of an object, with dedicated pprint-like format for lists. + """ + if isinstance(obj, list): + return '[' + (',\n '.join((_repr(e).replace('\n', '\n ') for e in obj))) + '\n]' + else: + return repr(obj) class Node(object): __slots__ = () """ Abstract base class for AST nodes. """ + def __repr__(self): + """ Generates a python representation of the current node + """ + result = self.__class__.__name__ + '(' + + indent = '' + separator = '' + for name in self.__slots__[:-2]: + result += separator + result += indent + result += name + '=' + (_repr(getattr(self, name)).replace('\n', '\n ' + (' ' * (len(name) + len(self.__class__.__name__))))) + + separator = ',' + indent = '\n ' + (' ' * len(self.__class__.__name__)) + + result += indent + ')' + + return result + def children(self): """ A sequence of all children that are Nodes """ @@ -112,21 +139,31 @@ * Modeled after Python's own AST visiting facilities (the ast module of Python 3.0) """ + + _method_cache = None + def visit(self, node): """ Visit a node. """ - method = 'visit_' + node.__class__.__name__ - visitor = getattr(self, method, self.generic_visit) + + if self._method_cache is None: + self._method_cache = {} + + visitor = self._method_cache.get(node.__class__.__name__, None) + if visitor is None: + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + self._method_cache[node.__class__.__name__] = visitor + return visitor(node) def generic_visit(self, node): """ Called if no explicit visitor function exists for a node. Implements preorder visiting of the node. """ - for c_name, c in node.children(): + for c in node: self.visit(c) - class ArrayDecl(Node): __slots__ = ('type', 'dim', 'dim_quals', 'coord', '__weakref__') def __init__(self, type, dim, dim_quals, coord=None): @@ -141,6 +178,12 @@ if self.dim is not None: nodelist.append(("dim", self.dim)) return tuple(nodelist) + def __iter__(self): + if self.type is not None: + yield self.type + if self.dim is not None: + yield self.dim + attr_names = ('dim_quals', ) class ArrayRef(Node): @@ -156,6 +199,12 @@ if self.subscript is not None: nodelist.append(("subscript", self.subscript)) return tuple(nodelist) + def __iter__(self): + if self.name is not None: + yield self.name + if self.subscript is not None: + yield self.subscript + attr_names = () class Assignment(Node): @@ -172,6 +221,12 @@ if self.rvalue is not None: nodelist.append(("rvalue", self.rvalue)) return tuple(nodelist) + def __iter__(self): + if self.lvalue is not None: + yield self.lvalue + if self.rvalue is not None: + yield self.rvalue + attr_names = ('op', ) class BinaryOp(Node): @@ -188,6 +243,12 @@ if self.right is not None: nodelist.append(("right", self.right)) return tuple(nodelist) + def __iter__(self): + if self.left is not None: + yield self.left + if self.right is not None: + yield self.right + attr_names = ('op', ) class Break(Node): @@ -198,6 +259,10 @@ def children(self): return () + def __iter__(self): + return + yield + attr_names = () class Case(Node): @@ -214,6 +279,12 @@ nodelist.append(("stmts[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + if self.expr is not None: + yield self.expr + for child in (self.stmts or []): + yield child + attr_names = () class Cast(Node): @@ -229,6 +300,12 @@ if self.expr is not None: nodelist.append(("expr", self.expr)) return tuple(nodelist) + def __iter__(self): + if self.to_type is not None: + yield self.to_type + if self.expr is not None: + yield self.expr + attr_names = () class Compound(Node): @@ -243,6 +320,10 @@ nodelist.append(("block_items[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.block_items or []): + yield child + attr_names = () class CompoundLiteral(Node): @@ -258,6 +339,12 @@ if self.init is not None: nodelist.append(("init", self.init)) return tuple(nodelist) + def __iter__(self): + if self.type is not None: + yield self.type + if self.init is not None: + yield self.init + attr_names = () class Constant(Node): @@ -271,6 +358,10 @@ nodelist = [] return tuple(nodelist) + def __iter__(self): + return + yield + attr_names = ('type', 'value', ) class Continue(Node): @@ -281,6 +372,10 @@ def children(self): return () + def __iter__(self): + return + yield + attr_names = () class Decl(Node): @@ -302,6 +397,14 @@ if self.bitsize is not None: nodelist.append(("bitsize", self.bitsize)) return tuple(nodelist) + def __iter__(self): + if self.type is not None: + yield self.type + if self.init is not None: + yield self.init + if self.bitsize is not None: + yield self.bitsize + attr_names = ('name', 'quals', 'storage', 'funcspec', ) class DeclList(Node): @@ -316,6 +419,10 @@ nodelist.append(("decls[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.decls or []): + yield child + attr_names = () class Default(Node): @@ -330,6 +437,10 @@ nodelist.append(("stmts[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.stmts or []): + yield child + attr_names = () class DoWhile(Node): @@ -345,6 +456,12 @@ if self.stmt is not None: nodelist.append(("stmt", self.stmt)) return tuple(nodelist) + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + attr_names = () class EllipsisParam(Node): @@ -355,6 +472,10 @@ def children(self): return () + def __iter__(self): + return + yield + attr_names = () class EmptyStatement(Node): @@ -365,6 +486,10 @@ def children(self): return () + def __iter__(self): + return + yield + attr_names = () class Enum(Node): @@ -379,6 +504,10 @@ if self.values is not None: nodelist.append(("values", self.values)) return tuple(nodelist) + def __iter__(self): + if self.values is not None: + yield self.values + attr_names = ('name', ) class Enumerator(Node): @@ -393,6 +522,10 @@ if self.value is not None: nodelist.append(("value", self.value)) return tuple(nodelist) + def __iter__(self): + if self.value is not None: + yield self.value + attr_names = ('name', ) class EnumeratorList(Node): @@ -407,6 +540,10 @@ nodelist.append(("enumerators[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.enumerators or []): + yield child + attr_names = () class ExprList(Node): @@ -421,6 +558,10 @@ nodelist.append(("exprs[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.exprs or []): + yield child + attr_names = () class FileAST(Node): @@ -435,6 +576,10 @@ nodelist.append(("ext[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.ext or []): + yield child + attr_names = () class For(Node): @@ -454,6 +599,16 @@ if self.stmt is not None: nodelist.append(("stmt", self.stmt)) return tuple(nodelist) + def __iter__(self): + if self.init is not None: + yield self.init + if self.cond is not None: + yield self.cond + if self.next is not None: + yield self.next + if self.stmt is not None: + yield self.stmt + attr_names = () class FuncCall(Node): @@ -469,6 +624,12 @@ if self.args is not None: nodelist.append(("args", self.args)) return tuple(nodelist) + def __iter__(self): + if self.name is not None: + yield self.name + if self.args is not None: + yield self.args + attr_names = () class FuncDecl(Node): @@ -484,6 +645,12 @@ if self.type is not None: nodelist.append(("type", self.type)) return tuple(nodelist) + def __iter__(self): + if self.args is not None: + yield self.args + if self.type is not None: + yield self.type + attr_names = () class FuncDef(Node): @@ -502,6 +669,14 @@ nodelist.append(("param_decls[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + if self.decl is not None: + yield self.decl + if self.body is not None: + yield self.body + for child in (self.param_decls or []): + yield child + attr_names = () class Goto(Node): @@ -514,6 +689,10 @@ nodelist = [] return tuple(nodelist) + def __iter__(self): + return + yield + attr_names = ('name', ) class ID(Node): @@ -526,6 +705,10 @@ nodelist = [] return tuple(nodelist) + def __iter__(self): + return + yield + attr_names = ('name', ) class IdentifierType(Node): @@ -538,6 +721,10 @@ nodelist = [] return tuple(nodelist) + def __iter__(self): + return + yield + attr_names = ('names', ) class If(Node): @@ -555,6 +742,14 @@ if self.iffalse is not None: nodelist.append(("iffalse", self.iffalse)) return tuple(nodelist) + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.iftrue is not None: + yield self.iftrue + if self.iffalse is not None: + yield self.iffalse + attr_names = () class InitList(Node): @@ -569,6 +764,10 @@ nodelist.append(("exprs[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.exprs or []): + yield child + attr_names = () class Label(Node): @@ -583,6 +782,10 @@ if self.stmt is not None: nodelist.append(("stmt", self.stmt)) return tuple(nodelist) + def __iter__(self): + if self.stmt is not None: + yield self.stmt + attr_names = ('name', ) class NamedInitializer(Node): @@ -599,6 +802,12 @@ nodelist.append(("name[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + if self.expr is not None: + yield self.expr + for child in (self.name or []): + yield child + attr_names = () class ParamList(Node): @@ -613,6 +822,10 @@ nodelist.append(("params[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.params or []): + yield child + attr_names = () class PtrDecl(Node): @@ -627,6 +840,10 @@ if self.type is not None: nodelist.append(("type", self.type)) return tuple(nodelist) + def __iter__(self): + if self.type is not None: + yield self.type + attr_names = ('quals', ) class Return(Node): @@ -640,6 +857,10 @@ if self.expr is not None: nodelist.append(("expr", self.expr)) return tuple(nodelist) + def __iter__(self): + if self.expr is not None: + yield self.expr + attr_names = () class Struct(Node): @@ -655,6 +876,10 @@ nodelist.append(("decls[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.decls or []): + yield child + attr_names = ('name', ) class StructRef(Node): @@ -671,6 +896,12 @@ if self.field is not None: nodelist.append(("field", self.field)) return tuple(nodelist) + def __iter__(self): + if self.name is not None: + yield self.name + if self.field is not None: + yield self.field + attr_names = ('type', ) class Switch(Node): @@ -686,6 +917,12 @@ if self.stmt is not None: nodelist.append(("stmt", self.stmt)) return tuple(nodelist) + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + attr_names = () class TernaryOp(Node): @@ -703,6 +940,14 @@ if self.iffalse is not None: nodelist.append(("iffalse", self.iffalse)) return tuple(nodelist) + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.iftrue is not None: + yield self.iftrue + if self.iffalse is not None: + yield self.iffalse + attr_names = () class TypeDecl(Node): @@ -718,6 +963,10 @@ if self.type is not None: nodelist.append(("type", self.type)) return tuple(nodelist) + def __iter__(self): + if self.type is not None: + yield self.type + attr_names = ('declname', 'quals', ) class Typedef(Node): @@ -734,6 +983,10 @@ if self.type is not None: nodelist.append(("type", self.type)) return tuple(nodelist) + def __iter__(self): + if self.type is not None: + yield self.type + attr_names = ('name', 'quals', 'storage', ) class Typename(Node): @@ -749,6 +1002,10 @@ if self.type is not None: nodelist.append(("type", self.type)) return tuple(nodelist) + def __iter__(self): + if self.type is not None: + yield self.type + attr_names = ('name', 'quals', ) class UnaryOp(Node): @@ -763,6 +1020,10 @@ if self.expr is not None: nodelist.append(("expr", self.expr)) return tuple(nodelist) + def __iter__(self): + if self.expr is not None: + yield self.expr + attr_names = ('op', ) class Union(Node): @@ -778,6 +1039,10 @@ nodelist.append(("decls[%d]" % i, child)) return tuple(nodelist) + def __iter__(self): + for child in (self.decls or []): + yield child + attr_names = ('name', ) class While(Node): @@ -793,5 +1058,27 @@ if self.stmt is not None: nodelist.append(("stmt", self.stmt)) return tuple(nodelist) + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + attr_names = () +class Pragma(Node): + __slots__ = ('string', 'coord', '__weakref__') + def __init__(self, string, coord=None): + self.string = string + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ('string', ) + diff --git a/lib_pypy/cffi/_pycparser/c_generator.py b/lib_pypy/cffi/_pycparser/c_generator.py --- a/lib_pypy/cffi/_pycparser/c_generator.py +++ b/lib_pypy/cffi/_pycparser/c_generator.py @@ -3,7 +3,7 @@ # # C code generator from pycparser AST nodes. # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #------------------------------------------------------------------------------ from . import c_ast @@ -40,6 +40,12 @@ def visit_ID(self, n): return n.name + def visit_Pragma(self, n): + ret = '#pragma' + if n.string: + ret += ' ' + n.string + return ret + def visit_ArrayRef(self, n): arrref = self._parenthesize_unless_simple(n.name) return arrref + '[' + self.visit(n.subscript) + ']' @@ -113,7 +119,7 @@ return s def visit_Cast(self, n): - s = '(' + self._generate_type(n.to_type) + ')' + s = '(' + self._generate_type(n.to_type, emit_declname=False) + ')' return s + ' ' + self._parenthesize_unless_simple(n.expr) def visit_ExprList(self, n): @@ -129,18 +135,20 @@ return ', '.join(visited_subexprs) def visit_Enum(self, n): - s = 'enum' - if n.name: s += ' ' + n.name - if n.values: - s += ' {' - for i, enumerator in enumerate(n.values.enumerators): - s += enumerator.name - if enumerator.value: - s += ' = ' + self.visit(enumerator.value) - if i != len(n.values.enumerators) - 1: - s += ', ' - s += '}' - return s + return self._generate_struct_union_enum(n, name='enum') + + def visit_Enumerator(self, n): + if not n.value: + return '{indent}{name},\n'.format( + indent=self._make_indent(), + name=n.name, + ) + else: + return '{indent}{name} = {value},\n'.format( + indent=self._make_indent(), + name=n.name, + value=self.visit(n.value), + ) def visit_FuncDef(self, n): decl = self.visit(n.decl) @@ -157,6 +165,8 @@ for ext in n.ext: if isinstance(ext, c_ast.FuncDef): s += self.visit(ext) + elif isinstance(ext, c_ast.Pragma): + s += self.visit(ext) + '\n' else: s += self.visit(ext) + ';\n' return s @@ -170,6 +180,10 @@ s += self._make_indent() + '}\n' return s + def visit_CompoundLiteral(self, n): + return '(' + self.visit(n.type) + '){' + self.visit(n.init) + '}' + + def visit_EmptyStatement(self, n): return ';' @@ -188,9 +202,9 @@ return 'continue;' def visit_TernaryOp(self, n): - s = self._visit_expr(n.cond) + ' ? ' - s += self._visit_expr(n.iftrue) + ' : ' - s += self._visit_expr(n.iffalse) + s = '(' + self._visit_expr(n.cond) + ') ? ' + s += '(' + self._visit_expr(n.iftrue) + ') : ' + s += '(' + self._visit_expr(n.iffalse) + ')' return s def visit_If(self, n): @@ -256,43 +270,67 @@ return '...' def visit_Struct(self, n): - return self._generate_struct_union(n, 'struct') + return self._generate_struct_union_enum(n, 'struct') def visit_Typename(self, n): return self._generate_type(n.type) def visit_Union(self, n): - return self._generate_struct_union(n, 'union') + return self._generate_struct_union_enum(n, 'union') def visit_NamedInitializer(self, n): s = '' for name in n.name: if isinstance(name, c_ast.ID): s += '.' + name.name - elif isinstance(name, c_ast.Constant): - s += '[' + name.value + ']' - s += ' = ' + self.visit(n.expr) + else: + s += '[' + self.visit(name) + ']' + s += ' = ' + self._visit_expr(n.expr) return s def visit_FuncDecl(self, n): return self._generate_type(n) - def _generate_struct_union(self, n, name): - """ Generates code for structs and unions. name should be either - 'struct' or union. + def visit_ArrayDecl(self, n): + return self._generate_type(n, emit_declname=False) + + def visit_TypeDecl(self, n): + return self._generate_type(n, emit_declname=False) + + def visit_PtrDecl(self, n): + return self._generate_type(n, emit_declname=False) + + def _generate_struct_union_enum(self, n, name): + """ Generates code for structs, unions, and enums. name should be + 'struct', 'union', or 'enum'. """ + if name in ('struct', 'union'): + members = n.decls + body_function = self._generate_struct_union_body + else: + assert name == 'enum' + members = None if n.values is None else n.values.enumerators + body_function = self._generate_enum_body s = name + ' ' + (n.name or '') - if n.decls: + if members is not None: + # None means no members + # Empty sequence means an empty list of members s += '\n' s += self._make_indent() self.indent_level += 2 s += '{\n' - for decl in n.decls: - s += self._generate_stmt(decl) + s += body_function(members) self.indent_level -= 2 s += self._make_indent() + '}' return s + def _generate_struct_union_body(self, members): + return ''.join(self._generate_stmt(decl) for decl in members) + + def _generate_enum_body(self, members): + # `[:-2] + '\n'` removes the final `,` from the enumerator list + return ''.join(self.visit(value) for value in members)[:-2] + '\n' + def _generate_stmt(self, n, add_indent=False): """ Generation from a statement node. This method exists as a wrapper for individual visit_* methods to handle different treatment of @@ -330,7 +368,7 @@ s += self._generate_type(n.type) return s - def _generate_type(self, n, modifiers=[]): + def _generate_type(self, n, modifiers=[], emit_declname = True): """ Recursive generation from a type node. n is the type node. modifiers collects the PtrDecl, ArrayDecl and FuncDecl modifiers encountered on the way down to a TypeDecl, to allow proper @@ -344,23 +382,29 @@ if n.quals: s += ' '.join(n.quals) + ' ' s += self.visit(n.type) - nstr = n.declname if n.declname else '' + nstr = n.declname if n.declname and emit_declname else '' # Resolve modifiers. # Wrap in parens to distinguish pointer to array and pointer to # function syntax. # for i, modifier in enumerate(modifiers): if isinstance(modifier, c_ast.ArrayDecl): - if (i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl)): - nstr = '(' + nstr + ')' - nstr += '[' + self.visit(modifier.dim) + ']' + if (i != 0 and + isinstance(modifiers[i - 1], c_ast.PtrDecl)): + nstr = '(' + nstr + ')' + nstr += '[' + if modifier.dim_quals: + nstr += ' '.join(modifier.dim_quals) + ' ' + nstr += self.visit(modifier.dim) + ']' elif isinstance(modifier, c_ast.FuncDecl): - if (i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl)): - nstr = '(' + nstr + ')' + if (i != 0 and + isinstance(modifiers[i - 1], c_ast.PtrDecl)): + nstr = '(' + nstr + ')' nstr += '(' + self.visit(modifier.args) + ')' elif isinstance(modifier, c_ast.PtrDecl): if modifier.quals: - nstr = '* %s %s' % (' '.join(modifier.quals), nstr) + nstr = '* %s%s' % (' '.join(modifier.quals), + ' ' + nstr if nstr else '') else: nstr = '*' + nstr if nstr: s += ' ' + nstr @@ -368,11 +412,12 @@ elif typ == c_ast.Decl: return self._generate_decl(n.type) elif typ == c_ast.Typename: - return self._generate_type(n.type) + return self._generate_type(n.type, emit_declname = emit_declname) elif typ == c_ast.IdentifierType: return ' '.join(n.names) + ' ' elif typ in (c_ast.ArrayDecl, c_ast.PtrDecl, c_ast.FuncDecl): - return self._generate_type(n.type, modifiers + [n]) + return self._generate_type(n.type, modifiers + [n], + emit_declname = emit_declname) else: return self.visit(n) @@ -395,5 +440,5 @@ """ Returns True for nodes that are "simple" - i.e. nodes that always have higher precedence than operators. """ - return isinstance(n,( c_ast.Constant, c_ast.ID, c_ast.ArrayRef, - c_ast.StructRef, c_ast.FuncCall)) + return isinstance(n, (c_ast.Constant, c_ast.ID, c_ast.ArrayRef, + c_ast.StructRef, c_ast.FuncCall)) diff --git a/lib_pypy/cffi/_pycparser/c_lexer.py b/lib_pypy/cffi/_pycparser/c_lexer.py --- a/lib_pypy/cffi/_pycparser/c_lexer.py +++ b/lib_pypy/cffi/_pycparser/c_lexer.py @@ -3,7 +3,7 @@ # # CLexer class: lexer for the C language # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #------------------------------------------------------------------------------ import re @@ -19,7 +19,7 @@ tokens. The public attribute filename can be set to an initial - filaneme, but the lexer will update it upon #line + filename, but the lexer will update it upon #line directives. """ def __init__(self, error_func, on_lbrace_func, on_rbrace_func, @@ -52,8 +52,8 @@ # Allow either "# line" or "# <num>" to support GCC's # cpp output # - self.line_pattern = re.compile('([ \t]*line\W)|([ \t]*\d+)') - self.pragma_pattern = re.compile('[ \t]*pragma\W') + self.line_pattern = re.compile(r'([ \t]*line\W)|([ \t]*\d+)') + self.pragma_pattern = re.compile(r'[ \t]*pragma\W') def build(self, **kwargs): """ Builds the lexer from the specification. Must be @@ -102,11 +102,11 @@ keywords = ( '_BOOL', '_COMPLEX', 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST', 'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE', 'ELSE', 'ENUM', 'EXTERN', - 'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG', + 'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG', 'REGISTER', 'OFFSETOF', 'RESTRICT', 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT', 'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID', - 'VOLATILE', 'WHILE', + 'VOLATILE', 'WHILE', '__INT128', ) keyword_map = {} @@ -171,7 +171,9 @@ 'ELLIPSIS', # pre-processor - 'PPHASH', # '#' + 'PPHASH', # '#' + 'PPPRAGMA', # 'pragma' + 'PPPRAGMASTR', ) ## @@ -203,12 +205,37 @@ # parse all correct code, even if it means to sometimes parse incorrect # code. # - simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" - decimal_escape = r"""(\d+)""" - hex_escape = r"""(x[0-9a-fA-F]+)""" - bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])""" + # The original regexes were taken verbatim from the C syntax definition, + # and were later modified to avoid worst-case exponential running time. + # + # simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" + # decimal_escape = r"""(\d+)""" + # hex_escape = r"""(x[0-9a-fA-F]+)""" + # bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])""" + # + # The following modifications were made to avoid the ambiguity that allowed backtracking: + # (https://github.com/eliben/pycparser/issues/61) + # + # - \x was removed from simple_escape, unless it was not followed by a hex digit, to avoid ambiguity with hex_escape. + # - hex_escape allows one or more hex characters, but requires that the next character(if any) is not hex + # - decimal_escape allows one or more decimal characters, but requires that the next character(if any) is not a decimal + # - bad_escape does not allow any decimals (8-9), to avoid conflicting with the permissive decimal_escape. + # + # Without this change, python's `re` module would recursively try parsing each ambiguous escape sequence in multiple ways. + # e.g. `\123` could be parsed as `\1`+`23`, `\12`+`3`, and `\123`. + + simple_escape = r"""([a-wyzA-Z._~!=&\^\-\\?'"]|x(?![0-9a-fA-F]))""" + decimal_escape = r"""(\d+)(?!\d)""" + hex_escape = r"""(x[0-9a-fA-F]+)(?![0-9a-fA-F])""" + bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-9])""" escape_sequence = r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))' + + # This complicated regex with lookahead might be slow for strings, so because all of the valid escapes (including \x) allowed + # 0 or more non-escaped characters after the first character, simple_escape+decimal_escape+hex_escape got simplified to + + escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])""" + cconst_char = r"""([^'\\\n]|"""+escape_sequence+')' char_const = "'"+cconst_char+"'" wchar_const = 'L'+char_const @@ -216,7 +243,7 @@ bad_char_const = r"""('"""+cconst_char+"""[^'\n]+')|('')|('"""+bad_escape+r"""[^'\n]*')""" # string literals (K&R2: A.2.6) - string_char = r"""([^"\\\n]|"""+escape_sequence+')' + string_char = r"""([^"\\\n]|"""+escape_sequence_start_in_string+')' string_literal = '"'+string_char+'*"' wstring_literal = 'L'+string_literal bad_string_literal = '"'+string_char+'*'+bad_escape+string_char+'*"' @@ -274,7 +301,6 @@ def t_ppline_NEWLINE(self, t): r'\n' - if self.pp_line is None: self._error('line number missing in #line', t) else: @@ -304,15 +330,14 @@ def t_pppragma_PPPRAGMA(self, t): r'pragma' - pass + return t - t_pppragma_ignore = ' \t<>.-{}();=+-*/$%@&^~!?:,0123456789' + t_pppragma_ignore = ' \t' - @TOKEN(string_literal) - def t_pppragma_STR(self, t): pass - - @TOKEN(identifier) - def t_pppragma_ID(self, t): pass + def t_pppragma_STR(self, t): + '.+' + t.type = 'PPPRAGMASTR' + return t def t_pppragma_error(self, t): self._error('invalid #pragma directive', t) @@ -482,4 +507,3 @@ def t_error(self, t): msg = 'Illegal character %s' % repr(t.value[0]) self._error(msg, t) - diff --git a/lib_pypy/cffi/_pycparser/c_parser.py b/lib_pypy/cffi/_pycparser/c_parser.py --- a/lib_pypy/cffi/_pycparser/c_parser.py +++ b/lib_pypy/cffi/_pycparser/c_parser.py @@ -3,7 +3,7 @@ # # CParser class: Parser and AST builder for the C language # -# Copyright (C) 2008-2015, Eli Bendersky +# Eli Bendersky [https://eli.thegreenplace.net/] # License: BSD #------------------------------------------------------------------------------ import re @@ -12,14 +12,16 @@ from . import c_ast from .c_lexer import CLexer -from .plyparser import PLYParser, Coord, ParseError +from .plyparser import PLYParser, Coord, ParseError, parameterized, template from .ast_transforms import fix_switch_cases +@template class CParser(PLYParser): def __init__( self, lex_optimize=True, + lexer=CLexer, lextab='cffi._pycparser.lextab', yacc_optimize=True, yacctab='cffi._pycparser.yacctab', @@ -42,6 +44,10 @@ to save the re-generation of the lexer table on each run. + lexer: + Set this parameter to define the lexer to use if + you're not using the default CLexer. + lextab: Points to the lex table that's used for optimized mode. Only if you're modifying the lexer and want @@ -70,7 +76,7 @@ Set this parameter to control the location of generated lextab and yacctab files. """ - self.clex = CLexer( + self.clex = lexer( error_func=self._lex_error_func, on_lbrace_func=self._lex_on_lbrace_func, _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
