Author: Antonio Cuni <anto.c...@gmail.com>
Branch: gc-disable
Changeset: r95200:f15f2aefb48e
Date: 2018-10-10 14:51 +0200
http://bitbucket.org/pypy/pypy/changeset/f15f2aefb48e/
Log: hg merge default
diff too long, truncating to 2000 out of 22739 lines
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,7 +33,12 @@
050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1
+a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2
fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -4,7 +4,7 @@
Welcome to PyPy!
-PyPy is an interperter that implements the Python programming language, based
+PyPy is an interpreter that implements the Python programming language, based
on the RPython compiler framework for dynamic language implementations.
The home page for the interpreter is:
@@ -15,29 +15,29 @@
http://doc.pypy.org/
-More documentation about the RPython framework can be found here
+More documentation about the RPython framework can be found here:
- http://rpython.readthedocs.io
+ http://rpython.readthedocs.io/
-The source for the documentation is in the pypy/doc directory
+The source for the documentation is in the pypy/doc directory.
+
Using PyPy instead of CPython
-=============================
+-----------------------------
-Please read the information at http://pypy.org to find the correct way to
+Please read the information at http://pypy.org/ to find the correct way to
download and use PyPy as an alternative to CPython.
+
Building
-========
+--------
Building PyPy is not the recommended way to obtain the PyPy alternative python
interpreter. It is time-consuming and requires significant computing resources.
-More information can be found here
+More information can be found here:
http://doc.pypy.org/en/latest/build.html
Enjoy and send us feedback!
the pypy-dev team <pypy-...@python.org>
-
-
diff --git a/lib-python/2.7/code.py b/lib-python/2.7/code.py
--- a/lib-python/2.7/code.py
+++ b/lib-python/2.7/code.py
@@ -104,6 +104,12 @@
except SystemExit:
raise
except:
+ if softspace(sys.stdout, 0):
+ print
+ try:
+ sys.stdout.flush()
+ except:
+ pass
self.showtraceback()
else:
if softspace(sys.stdout, 0):
diff --git a/lib-python/2.7/fractions.py b/lib-python/2.7/fractions.py
--- a/lib-python/2.7/fractions.py
+++ b/lib-python/2.7/fractions.py
@@ -517,8 +517,13 @@
# Get integers right.
return hash(self._numerator)
# Expensive check, but definitely correct.
- if self == float(self):
- return hash(float(self))
+ # PyPy: the following 4 lines used to be almost twice slower:
+ # if self == float(self):
+ # return hash(float(self))
+ f = float(self)
+ x, y = f.as_integer_ratio() # signs are correct: y is positive
+ if self._numerator == x and self._denominator == y:
+ return hash(f)
else:
# Use tuple's hash to avoid a high collision rate on
# simple fractions.
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -136,9 +136,14 @@
__get_hash = __get_openssl_constructor
algorithms_available = algorithms_available.union(
_hashlib.openssl_md_meth_names)
-except ImportError:
+except ImportError as e:
new = __py_new
__get_hash = __get_builtin_constructor
+ # added by PyPy
+ import warnings
+ warnings.warn("The _hashlib module is not available, falling back "
+ "to a much slower implementation (%s)" % str(e),
+ RuntimeWarning)
for __func_name in __always_supported:
# try them all, some may not work due to the OpenSSL
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -194,5 +194,6 @@
def_op('CALL_METHOD', 202) # #args not including 'self'
def_op('BUILD_LIST_FROM_ARG', 203)
jrel_op('JUMP_IF_NOT_DEBUG', 204) # jump over assert statements
+def_op('LOAD_REVDB_VAR', 205) # reverse debugger (syntax example: $5)
del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py
--- a/lib-python/2.7/shutil.py
+++ b/lib-python/2.7/shutil.py
@@ -396,17 +396,21 @@
return archive_name
-def _call_external_zip(base_dir, zip_filename, verbose=False, dry_run=False):
+def _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger):
# XXX see if we want to keep an external call here
if verbose:
zipoptions = "-r"
else:
zipoptions = "-rq"
- from distutils.errors import DistutilsExecError
- from distutils.spawn import spawn
+ cmd = ["zip", zipoptions, zip_filename, base_dir]
+ if logger is not None:
+ logger.info(' '.join(cmd))
+ if dry_run:
+ return
+ import subprocess
try:
- spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run)
- except DistutilsExecError:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
# XXX really should distinguish between "couldn't find
# external 'zip' command" and "zip failed".
raise ExecError, \
@@ -440,7 +444,7 @@
zipfile = None
if zipfile is None:
- _call_external_zip(base_dir, zip_filename, verbose, dry_run)
+ _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger)
else:
if logger is not None:
logger.info("creating '%s' and adding '%s' to it",
diff --git a/lib-python/2.7/test/test_inspect.py
b/lib-python/2.7/test/test_inspect.py
--- a/lib-python/2.7/test/test_inspect.py
+++ b/lib-python/2.7/test/test_inspect.py
@@ -45,6 +45,9 @@
git = mod.StupidGit()
+class ExampleClassWithSlot(object):
+ __slots__ = 'myslot'
+
class IsTestBase(unittest.TestCase):
predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode,
inspect.isframe, inspect.isfunction, inspect.ismethod,
@@ -96,7 +99,11 @@
else:
self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals))
if hasattr(types, 'MemberDescriptorType'):
- self.istest(inspect.ismemberdescriptor, 'type(lambda:
None).func_globals')
+ # App-level slots are member descriptors on both PyPy and
+ # CPython, but the various built-in attributes are all
+ # getsetdescriptors on PyPy. So check ismemberdescriptor()
+ # with an app-level slot.
+ self.istest(inspect.ismemberdescriptor,
'ExampleClassWithSlot.myslot')
else:
self.assertFalse(inspect.ismemberdescriptor(type(lambda:
None).func_globals))
diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -83,9 +83,19 @@
DictProxyType = type(TypeType.__dict__)
NotImplementedType = type(NotImplemented)
-# For Jython, the following two types are identical
+#
+# On CPython, FunctionType.__code__ is a 'getset_descriptor', but
+# FunctionType.__globals__ is a 'member_descriptor', just like app-level
+# slots. On PyPy, all descriptors of built-in types are
+# 'getset_descriptor', but the app-level slots are 'member_descriptor'
+# as well. (On Jython the situation might still be different.)
+#
+# Note that MemberDescriptorType was equal to GetSetDescriptorType in
+# PyPy <= 6.0.
+#
GetSetDescriptorType = type(FunctionType.func_code)
-MemberDescriptorType = type(FunctionType.func_globals)
+class _C(object): __slots__ = 's'
+MemberDescriptorType = type(_C.s)
del sys, _f, _g, _C, _x # Not for export
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -183,6 +183,7 @@
self._buffer = self._ffiarray(self._length_, autofree=True)
for i, arg in enumerate(args):
self[i] = arg
+ _init_no_arg_ = __init__
def _fix_index(self, index):
if index < 0:
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -120,7 +120,7 @@
raise ValueError(
"Buffer size too small (%d instead of at least %d bytes)"
% (len(buf) + offset, size + offset))
- result = self()
+ result = self._newowninstance_()
dest = result._buffer.buffer
try:
raw_addr = buf._pypy_raw_address()
@@ -131,6 +131,11 @@
memmove(dest, raw_addr, size)
return result
+ def _newowninstance_(self):
+ result = self.__new__(self)
+ result._init_no_arg_()
+ return result
+
class CArgObject(object):
""" simple wrapper around buffer, just for the case of freeing
@@ -162,6 +167,7 @@
def __init__(self, *args, **kwds):
raise TypeError("%s has no type" % (type(self),))
+ _init_no_arg_ = __init__
def _ensure_objects(self):
if '_objects' not in self.__dict__:
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -268,6 +268,7 @@
return
raise TypeError("Unknown constructor %s" % (args,))
+ _init_no_arg_ = __init__
def _wrap_callable(self, to_call, argtypes):
def f(*args):
@@ -485,6 +486,8 @@
return cobj, cobj._to_ffi_param(), type(cobj)
def _convert_args_for_callback(self, argtypes, args):
+ from _ctypes.structure import StructOrUnion
+ #
assert len(argtypes) == len(args)
newargs = []
for argtype, arg in zip(argtypes, args):
@@ -494,6 +497,10 @@
param = param._get_buffer_value()
elif self._is_primitive(argtype):
param = param.value
+ elif isinstance(param, StructOrUnion): # not a *pointer* to
struct
+ newparam = StructOrUnion.__new__(type(param))
+ param._copy_to(newparam._buffer.buffer)
+ param = newparam
newargs.append(param)
return newargs
@@ -557,7 +564,7 @@
keepalive, newarg, newargtype =
self._conv_param(argtype, defval)
else:
import ctypes
- val = argtype._type_()
+ val = argtype._type_._newowninstance_()
keepalive = None
newarg = ctypes.byref(val)
newargtype = type(newarg)
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -67,8 +67,11 @@
self._buffer = ffiarray(1, autofree=True)
if value is not None:
self.contents = value
+ def _init_no_arg_(self):
+ self._buffer = ffiarray(1, autofree=True)
self._ffiarray = ffiarray
self.__init__ = __init__
+ self._init_no_arg_ = _init_no_arg_
self._type_ = TP
def _build_ffiargtype(self):
@@ -137,27 +140,21 @@
if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()):
raise TypeError("cast() argument 2 must be a pointer type, not %s"
% (tp,))
+ result = tp._newowninstance_()
if isinstance(obj, (int, long)):
- result = tp()
result._buffer[0] = obj
return result
elif obj is None:
- result = tp()
return result
elif isinstance(obj, Array):
- ptr = tp.__new__(tp)
- ptr._buffer = tp._ffiarray(1, autofree=True)
- ptr._buffer[0] = obj._buffer
- result = ptr
+ result._buffer[0] = obj._buffer
elif isinstance(obj, bytes):
- result = tp()
result._buffer[0] = buffer(obj)._pypy_raw_address()
return result
elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
raise TypeError("cast() argument 1 must be a pointer, not %s"
% (type(obj),))
else:
- result = tp()
result._buffer[0] = obj._buffer[0]
# The casted objects '_objects' member:
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -390,11 +390,14 @@
self._buffer = self._ffiarray(1, autofree=True)
if value is not DEFAULT_VALUE:
self.value = value
+ _init_no_arg_ = __init__
def _ensure_objects(self):
- if self._type_ not in 'zZP':
- assert self._objects is None
- return self._objects
+ # No '_objects' is the common case for primitives. Examples
+ # where there is an _objects is if _type in 'zZP', or if
+ # self comes from 'from_buffer(buf)'. See module/test_lib_pypy/
+ # ctypes_test/test_buffers.py: test_from_buffer_keepalive.
+ return getattr(self, '_objects', None)
def _getvalue(self):
return self._buffer[0]
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -281,6 +281,7 @@
self.__setattr__(name, arg)
for name, arg in kwds.items():
self.__setattr__(name, arg)
+ _init_no_arg_ = __init__
def _subarray(self, fieldtype, name):
"""Return a _rawffi array of length 1 whose address is the same as
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.11.5
+Version: 1.12.0
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI
from .error import CDefError, FFIError, VerificationError, VerificationMissing
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
--- a/lib_pypy/cffi/_cffi_errors.h
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -50,7 +50,9 @@
"import sys\n"
"class FileLike:\n"
" def write(self, x):\n"
- " of.write(x)\n"
+ " try:\n"
+ " of.write(x)\n"
+ " except: pass\n"
" self.buf += x\n"
"fl = FileLike()\n"
"fl.buf = ''\n"
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,20 +8,43 @@
the same works for the other two macros. Py_DEBUG implies them,
but not the other way around.
- Issue #350 is still open: on Windows, the code here causes it to link
- with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was
- attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
- does not make PYTHON3.DLL available, and so the "correctly" compiled
- version would not run inside a virtualenv. We will re-apply the fix
- after virtualenv has been fixed for some time. For explanation, see
- issue #355. For a workaround if you want PYTHON3.DLL and don't worry
- about virtualenv, see issue #350. See also 'py_limited_api' in
- setuptools_ext.py.
+ The implementation is messy (issue #350): on Windows, with _MSC_VER,
+ we have to define Py_LIMITED_API even before including pyconfig.h.
+ In that case, we guess what pyconfig.h will do to the macros above,
+ and check our guess after the #include.
+
+ Note that on Windows, with CPython 3.x, you need virtualenv version
+ >= 16.0.0. Older versions don't copy PYTHON3.DLL. As a workaround
+ you can remove the definition of Py_LIMITED_API here.
+
+ See also 'py_limited_api' in cffi/setuptools_ext.py.
*/
#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-# include <pyconfig.h>
-# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-# define Py_LIMITED_API
+# ifdef _MSC_VER
+# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) &&
!defined(Py_REF_DEBUG)
+# define Py_LIMITED_API
+# endif
+# include <pyconfig.h>
+ /* sanity-check: Py_LIMITED_API will cause crashes if any of these
+ are also defined. Normally, the Python file PC/pyconfig.h does not
+ cause any of these to be defined, with the exception that _DEBUG
+ causes Py_DEBUG. Double-check that. */
+# ifdef Py_LIMITED_API
+# if defined(Py_DEBUG)
+# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API
is set"
+# endif
+# if defined(Py_TRACE_REFS)
+# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but
Py_LIMITED_API is set"
+# endif
+# if defined(Py_REF_DEBUG)
+# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but
Py_LIMITED_API is set"
+# endif
+# endif
+# else
+# include <pyconfig.h>
+# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+# define Py_LIMITED_API
+# endif
# endif
#endif
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.11.5"
+ "\ncompiled with cffi version: 1.12.0"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
self.CData, self.CType = backend._get_types()
self.buffer = backend.buffer
- def cdef(self, csource, override=False, packed=False):
+ def cdef(self, csource, override=False, packed=False, pack=None):
"""Parse the given C source. This registers all declared functions,
types, and global variables. The functions and global variables can
then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
The types can be used in 'ffi.new()' and other functions.
If 'packed' is specified as True, all structs declared inside this
cdef are packed, i.e. laid out without any field alignment at all.
+ Alternatively, 'pack' can be a small integer, and requests for
+ alignment greater than that are ignored (pack=1 is equivalent to
+ packed=True).
"""
- self._cdef(csource, override=override, packed=packed)
+ self._cdef(csource, override=override, packed=packed, pack=pack)
- def embedding_api(self, csource, packed=False):
- self._cdef(csource, packed=packed, dllexport=True)
+ def embedding_api(self, csource, packed=False, pack=None):
+ self._cdef(csource, packed=packed, pack=pack, dllexport=True)
if self._embedding is None:
self._embedding = ''
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -636,6 +636,10 @@
if isinstance(init, bytes):
init = [init[i:i+1] for i in range(len(init))]
else:
+ if isinstance(init, CTypesGenericArray):
+ if (len(init) != len(blob) or
+ not isinstance(init, CTypesArray)):
+ raise TypeError("length/type mismatch: %s" %
(init,))
init = tuple(init)
if len(init) > len(blob):
raise IndexError("too many initializers")
@@ -730,7 +734,8 @@
return self._new_struct_or_union('union', name, ctypes.Union)
def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
- totalsize=-1, totalalignment=-1, sflags=0):
+ totalsize=-1, totalalignment=-1, sflags=0,
+ pack=0):
if totalsize >= 0 or totalalignment >= 0:
raise NotImplementedError("the ctypes backend of CFFI does not
support "
"structures completed by verify();
please "
@@ -751,6 +756,8 @@
bfield_types[fname] = Ellipsis
if sflags & 8:
struct_or_union._pack_ = 1
+ elif pack:
+ struct_or_union._pack_ = pack
struct_or_union._fields_ = cfields
CTypesStructOrUnion._bfield_types = bfield_types
#
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
msg = 'parse error\n%s' % (msg,)
raise CDefError(msg)
- def parse(self, csource, override=False, packed=False, dllexport=False):
+ def parse(self, csource, override=False, packed=False, pack=None,
+ dllexport=False):
+ if packed:
+ if packed != True:
+ raise ValueError("'packed' should be False or True; use "
+ "'pack' to give another value")
+ if pack:
+ raise ValueError("cannot give both 'pack' and 'packed'")
+ pack = 1
+ elif pack:
+ if pack & (pack - 1):
+ raise ValueError("'pack' must be a power of two, not %r" %
+ (pack,))
+ else:
+ pack = 0
prev_options = self._options
try:
self._options = {'override': override,
- 'packed': packed,
+ 'packed': pack,
'dllexport': dllexport}
self._internal_parse(csource)
finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
fixedlayout = None
completed = 0
partial = False
- packed = False
+ packed = 0
def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
self.name = name
@@ -414,11 +414,14 @@
fldtypes = [tp.get_cached_btype(ffi, finishlist)
for tp in self.fldtypes]
lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
- sflags = 0
+ extra_flags = ()
if self.packed:
- sflags = 8 # SF_PACKED
+ if self.packed == 1:
+ extra_flags = (8,) # SF_PACKED
+ else:
+ extra_flags = (0, self.packed)
ffi._backend.complete_struct_or_union(BType, lst, self,
- -1, -1, sflags)
+ -1, -1, *extra_flags)
#
else:
fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
else:
flags.append("_CFFI_F_CHECK_FIELDS")
if tp.packed:
+ if tp.packed > 1:
+ raise NotImplementedError(
+ "%r is declared with 'pack=%r'; only 0 or 1 are "
+ "supported in API mode (try to use \"...;\", which "
+ "does not require a 'pack' declaration)" %
+ (tp, tp.packed))
flags.append("_CFFI_F_PACKED")
else:
flags.append("_CFFI_F_EXTERNAL")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,13 +81,8 @@
it doesn't so far, creating troubles. That's why we check
for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
-
- On Windows, it's better not to use py_limited_api until issue #355
- can be resolved (by having virtualenv copy PYTHON3.DLL). See also
- the start of _cffi_include.h.
"""
- if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
- and sys.platform != 'win32'):
+ if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
import setuptools
try:
setuptools_major_version =
int(setuptools.__version__.partition('.')[0])
@@ -167,6 +162,17 @@
module_path = module_name.split('.')
module_path[-1] += '.py'
generate_mod(os.path.join(self.build_lib, *module_path))
+ def get_source_files(self):
+ # This is called from 'setup.py sdist' only. Exclude
+ # the generate .py module in this case.
+ saved_py_modules = self.py_modules
+ try:
+ if saved_py_modules:
+ self.py_modules = [m for m in saved_py_modules
+ if m != module_name]
+ return base_class.get_source_files(self)
+ finally:
+ self.py_modules = saved_py_modules
dist.cmdclass['build_py'] = build_py_make_mod
# distutils and setuptools have no notion I could find of a
@@ -176,6 +182,7 @@
# the module. So we add it here, which gives a few apparently
# harmless warnings about not finding the file outside the
# build directory.
+ # Then we need to hack more in get_source_files(); see above.
if dist.py_modules is None:
dist.py_modules = []
dist.py_modules.append(module_name)
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -4,6 +4,8 @@
from _pwdgrp_cffi import ffi, lib
import _structseq
+import thread
+_lock = thread.allocate_lock()
try: from __pypy__ import builtinify
except ImportError: builtinify = lambda f: f
@@ -33,32 +35,35 @@
@builtinify
def getgrgid(gid):
- res = lib.getgrgid(gid)
- if not res:
- # XXX maybe check error eventually
- raise KeyError(gid)
- return _group_from_gstruct(res)
+ with _lock:
+ res = lib.getgrgid(gid)
+ if not res:
+ # XXX maybe check error eventually
+ raise KeyError(gid)
+ return _group_from_gstruct(res)
@builtinify
def getgrnam(name):
if not isinstance(name, basestring):
raise TypeError("expected string")
name = str(name)
- res = lib.getgrnam(name)
- if not res:
- raise KeyError("'getgrnam(): name not found: %s'" % name)
- return _group_from_gstruct(res)
+ with _lock:
+ res = lib.getgrnam(name)
+ if not res:
+ raise KeyError("'getgrnam(): name not found: %s'" % name)
+ return _group_from_gstruct(res)
@builtinify
def getgrall():
- lib.setgrent()
lst = []
- while 1:
- p = lib.getgrent()
- if not p:
- break
- lst.append(_group_from_gstruct(p))
- lib.endgrent()
+ with _lock:
+ lib.setgrent()
+ while 1:
+ p = lib.getgrent()
+ if not p:
+ break
+ lst.append(_group_from_gstruct(p))
+ lib.endgrent()
return lst
__all__ = ('struct_group', 'getgrgid', 'getgrnam', 'getgrall')
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -12,6 +12,8 @@
from _pwdgrp_cffi import ffi, lib
import _structseq
+import thread
+_lock = thread.allocate_lock()
try: from __pypy__ import builtinify
except ImportError: builtinify = lambda f: f
@@ -55,10 +57,11 @@
Return the password database entry for the given numeric user ID.
See pwd.__doc__ for more on password database entries.
"""
- pw = lib.getpwuid(uid)
- if not pw:
- raise KeyError("getpwuid(): uid not found: %s" % uid)
- return _mkpwent(pw)
+ with _lock:
+ pw = lib.getpwuid(uid)
+ if not pw:
+ raise KeyError("getpwuid(): uid not found: %s" % uid)
+ return _mkpwent(pw)
@builtinify
def getpwnam(name):
@@ -71,10 +74,11 @@
if not isinstance(name, basestring):
raise TypeError("expected string")
name = str(name)
- pw = lib.getpwnam(name)
- if not pw:
- raise KeyError("getpwname(): name not found: %s" % name)
- return _mkpwent(pw)
+ with _lock:
+ pw = lib.getpwnam(name)
+ if not pw:
+ raise KeyError("getpwname(): name not found: %s" % name)
+ return _mkpwent(pw)
@builtinify
def getpwall():
@@ -84,13 +88,14 @@
See pwd.__doc__ for more on password database entries.
"""
users = []
- lib.setpwent()
- while True:
- pw = lib.getpwent()
- if not pw:
- break
- users.append(_mkpwent(pw))
- lib.endpwent()
+ with _lock:
+ lib.setpwent()
+ while True:
+ pw = lib.getpwent()
+ if not pw:
+ break
+ users.append(_mkpwent(pw))
+ lib.endpwent()
return users
__all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall')
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,14 +39,10 @@
"_csv", "_cppyy", "_pypyjson", "_jitlog"
])
-from rpython.jit.backend import detect_cpu
-try:
- if detect_cpu.autodetect().startswith('x86'):
- if not sys.platform.startswith('openbsd'):
- working_modules.add('_vmprof')
- working_modules.add('faulthandler')
-except detect_cpu.ProcessorAutodetectError:
- pass
+import rpython.rlib.rvmprof.cintf
+if rpython.rlib.rvmprof.cintf.IS_SUPPORTED:
+ working_modules.add('_vmprof')
+ working_modules.add('faulthandler')
translation_modules = default_modules.copy()
translation_modules.update([
@@ -57,6 +53,11 @@
"termios", "_minimal_curses",
])
+reverse_debugger_disable_modules = set([
+ "_continuation", "_vmprof", "_multiprocessing",
+ "micronumpy",
+ ])
+
# XXX this should move somewhere else, maybe to platform ("is this posixish"
# check or something)
if sys.platform == "win32":
@@ -292,6 +293,9 @@
modules = working_modules.copy()
if config.translation.sandbox:
modules = default_modules
+ if config.translation.reverse_debugger:
+ for mod in reverse_debugger_disable_modules:
+ setattr(config.objspace.usemodules, mod, False)
# ignore names from 'essential_modules', notably 'exceptions', which
# may not be present in config.objspace.usemodules at all
modules = [name for name in modules if name not in essential_modules]
@@ -310,3 +314,4 @@
parser = to_optparse(config) #, useoptions=["translation.*"])
option, args = parser.parse_args()
print config
+ print working_modules
diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt
b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt
b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt
b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt
b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt
b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt
b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt
b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt
b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt
b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt
b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt
b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt
b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -323,7 +323,8 @@
-------------
* Hash randomization (``-R``) `is ignored in PyPy`_. In CPython
- before 3.4 it has `little point`_.
+ before 3.4 it has `little point`_. Both CPython >= 3.4 and PyPy3
+ implement the randomized SipHash algorithm and ignore ``-R``.
* You can't store non-string keys in type objects. For example::
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -20,7 +20,7 @@
OS and architecture. You may be able to use either use the
`most recent release`_ or one of our `development nightly build`_. These
builds depend on dynamically linked libraries that may not be available on your
-OS. See the section about `Linux binaries` for more info and alternatives that
+OS. See the section about `Linux binaries`_ for more info and alternatives that
may work on your system.
Please note that the nightly builds are not
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -3,6 +3,11 @@
PyPy's sandboxing features
==========================
+.. warning:: This is not actively maintained. You will likely have to fix
+ some issues yourself, or otherwise play around on your own. We provide
+ this documentation for historical reasions, it will not translate or
+ run on the latest PyPy code base.
+
Introduction
------------
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,12 +18,13 @@
'Antonio Cuni': ['antocuni', 'anto'],
'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
'Maciej Fijalkowski': ['fijal'],
- 'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
+ 'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf',
'cbolz'],
'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
- 'Richard Plangger':['planrich'],
- 'Michael Hudson': ['mwh'],
+ 'Richard Plangger': ['planrich', 'plan_rich'],
+ 'Remi Meier': ['remi'],
+ 'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'],
'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
- "Amaury Forgeot d'Arc": ['afa'],
+ "Amaury Forgeot d'Arc": ['afa', 'amaur...@gmail.com'],
'Alex Gaynor': ['alex', 'agaynor'],
'David Schneider': ['bivab', 'david'],
'Christian Tismer': ['chris', 'christian', 'tismer',
@@ -41,7 +42,7 @@
'Mark Pearse': ['mwp'],
'Toon Verwaest': ['tverwaes'],
'Eric van Riet Paap': ['ericvrp'],
- 'Jacob Hallen': ['jacob', 'jakob'],
+ 'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'],
'Anders Lehmann': ['ale', 'anders'],
'Bert Freudenberg': ['bert'],
'Boris Feigin': ['boris', 'boria'],
@@ -69,19 +70,25 @@
'Manuel Jacob': ['mjacob'],
'Rami Chowdhury': ['necaris'],
'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
- 'Wenzhu Man':['wenzhu man', 'wenzhuman'],
- 'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
- 'Richard Lancaster':['richardlancaster'],
- 'William Leslie':['William ML Leslie'],
- 'Spenser Bauman':['Spenser Andrew Bauman'],
- 'Raffael Tfirst':['raffael.tfi...@gmail.com'],
- 'timo':['t...@eistee.fritz.box'],
- 'Jasper Schulz':['Jasper.Schulz', 'jbs'],
- 'Aaron Gallagher':['"Aaron Gallagher'],
- 'Yasir Suhail':['yasirs'],
+ 'Wenzhu Man': ['wenzhu man', 'wenzhuman'],
+ 'Anton Gulenko': ['anton gulenko', 'anton_gulenko'],
+ 'Richard Lancaster': ['richardlancaster'],
+ 'William Leslie': ['William ML Leslie'],
+ 'Spenser Bauman': ['Spenser Andrew Bauman'],
+ 'Raffael Tfirst': ['raffael.tfi...@gmail.com'],
+ 'timo': ['t...@eistee.fritz.box'],
+ 'Jasper Schulz': ['Jasper.Schulz', 'jbs'],
+ 'Aaron Gallagher': ['"Aaron Gallagher'],
+ 'Yasir Suhail': ['yasirs'],
'Squeaky': ['squeaky'],
- "Amaury Forgeot d'Arc": ['amaur...@gmail.com'],
"Dodan Mihai": ['mihai.do...@gmail.com'],
+ 'Wim Lavrijsen': ['wlav'],
+ 'Toon Verwaest': ['toon', 'tverwaes'],
+ 'Seo Sanghyeon': ['sanxiyn'],
+ 'Leonardo Santagada': ['santagada'],
+ 'Laurence Tratt': ['ltratt'],
+ 'Pieter Zieschang': ['pzieschang', 'p_ziesch...@yahoo.de'],
+ 'John Witulski': ['witulski'],
}
alias_map = {}
@@ -103,7 +110,8 @@
return set()
ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
'in-progress', 'bits of', 'even a little', 'floating',
- 'a bit', 'reviewing']
+ 'a bit', 'reviewing', 'looking', 'advising', 'partly',
'ish',
+ 'watching', 'mostly', 'jumping']
sep_words = ['and', ';', '+', '/', 'with special by']
nicknames = match.group(1)
for word in ignore_words:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,9 +7,13 @@
.. branch: cppyy-packaging
-Upgrade to backend 0.6.0, support exception handling from wrapped functions,
-update enum handling, const correctness for data members and associated tests,
-support anonymous enums, support for function pointer arguments
+Main items: vastly better template resolution and improved performance. In
+detail: upgrade to backend 1.4, improved handling of templated methods and
+functions (in particular automatic deduction of types), improved pythonization
+interface, range of compatibility fixes for Python3, free functions now take
+fast libffi path when possible, moves for strings (incl. from Python str),
+easier/faster handling of std::vector by numpy, improved and faster object
+identity preservation
.. branch: socket_default_timeout_blockingness
@@ -18,3 +22,20 @@
.. branch: crypt_h
Include crypt.h for crypt() on Linux
+
+.. branch: gc-more-logging
+
+Log additional gc-minor and gc-collect-step info in the PYPYLOG
+
+.. branch: reverse-debugger
+
+The reverse-debugger branch has been merged. For more information, see
+https://bitbucket.org/pypy/revdb
+
+
+.. branch: pyparser-improvements-3
+
+Small refactorings in the Python parser.
+
+.. branch: fix-readme-typo
+
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -29,29 +29,28 @@
``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for
Python``
or in
``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``.
-A current version of ``setuptools`` will be able to find it there. For
-Windows 10, you must right-click the download, and under ``Properties`` ->
-``Compatibility`` mark it as ``Run run this program in comatibility mode for``
-``Previous version...``. Also, you must download and install the ``.Net
Framework 3.5``,
+A current version of ``setuptools`` will be able to find it there.
+Also, you must download and install the ``.Net Framework 3.5``,
otherwise ``mt.exe`` will silently fail. Installation will begin automatically
by running the mt.exe command by hand from a DOS window (that is how the author
discovered the problem).
.. _Microsoft Visual C++ Compiler for Python 2.7:
https://www.microsoft.com/EN-US/DOWNLOAD/DETAILS.ASPX?ID=44266
-Installing "Build Tools for Visual Studio 2017" (for Python 3)
+Installing "Build Tools for Visual Studio 2015" (for Python 3)
--------------------------------------------------------------
-As documented in the CPython Wiki_, CPython now recommends Visual C++ version
-14.0. A compact version of the compiler suite can be obtained from Microsoft_
-downloads, search the page for "Build Tools for Visual Studio 2017".
+As documented in the CPython Wiki_, CPython recommends Visual C++ version
+14.0 for python version 3.5. A compact version of the compiler suite can be
+obtained from Microsoft_ downloads, search the page for "Microsoft Build Tools
2015".
-You will also need to install the the `Windows SDK`_ in order to use the
-`mt.exe` mainfest compiler.
+You will need to reboot the computer for the installation to successfully
install and
+run the `mt.exe` mainfest compiler. The installation will set the
+`VS140COMNTOOLS` environment variable, this is key to distutils/setuptools
+finding the compiler
.. _Wiki: https://wiki.python.org/moin/WindowsCompilers
-.. _Microsoft: https://www.visualstudio.com/downloads
-.. _`Windows SDK`:
https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk
+.. _Microsoft: https://www.visualstudio.com/vs/older-downloads/
Translating PyPy with Visual Studio
-----------------------------------
@@ -99,6 +98,9 @@
Setting Up Visual Studio 9.0 for building SSL in Python3
--------------------------------------------------------
+**Note: this is old information, left for historical reference. We recommend
+using Visual Studio 2015, which now seems to properly set this all up.**
+
On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step
after
translation. However ``distutils`` does not support the Micorosft-provided
Visual C
compiler, and ``cffi`` depends on ``distutils`` to find the compiler. The
@@ -146,14 +148,14 @@
Installing external packages
----------------------------
-We uses a `repository` parallel to pypy to hold binary compiled versions of the
+We uses a subrepository_ inside pypy to hold binary compiled versions of the
build dependencies for windows. As part of the `rpython` setup stage,
environment
variables will be set to use these dependencies. The repository has a README
file on how to replicate, and a branch for each supported platform. You may run
the `get_externals.py` utility to checkout the proper branch for your platform
and PyPy version.
-.. _repository: https://bitbucket.org/pypy/external
+.. _subrepository: https://bitbucket.org/pypy/external
Using the mingw compiler
------------------------
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -83,11 +83,24 @@
sys.excepthook(), catching SystemExit, printing a newline after
sys.stdout if needed, etc.
"""
+ # don't use try:except: here, otherwise the exception remains
+ # visible in user code. Make sure revdb_stop is a callable, so
+ # that we can call it immediately after finally: below. Doing
+ # so minimizes the number of "blind" lines that we need to go
+ # back from, with "bstep", after we do "continue" in revdb.
+ if '__pypy__' in sys.builtin_module_names:
+ from __pypy__ import revdb_stop
+ else:
+ revdb_stop = None
+ if revdb_stop is None:
+ revdb_stop = lambda: None
+
try:
# run it
try:
f(*fargs, **fkwds)
finally:
+ revdb_stop()
sys.settrace(None)
sys.setprofile(None)
diff --git a/pypy/interpreter/astcompiler/assemble.py
b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -673,6 +673,7 @@
ops.JUMP_IF_NOT_DEBUG: 0,
ops.BUILD_LIST_FROM_ARG: 1,
+ ops.LOAD_REVDB_VAR: 1,
}
diff --git a/pypy/interpreter/astcompiler/ast.py
b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1593,6 +1593,8 @@
return Num.from_object(space, w_node)
if space.isinstance_w(w_node, get(space).w_Str):
return Str.from_object(space, w_node)
+ if space.isinstance_w(w_node, get(space).w_RevDBMetaVar):
+ return RevDBMetaVar.from_object(space, w_node)
if space.isinstance_w(w_node, get(space).w_Attribute):
return Attribute.from_object(space, w_node)
if space.isinstance_w(w_node, get(space).w_Subscript):
@@ -2456,6 +2458,41 @@
State.ast_type('Str', 'expr', ['s'])
+class RevDBMetaVar(expr):
+
+ def __init__(self, metavar, lineno, col_offset):
+ self.metavar = metavar
+ expr.__init__(self, lineno, col_offset)
+
+ def walkabout(self, visitor):
+ visitor.visit_RevDBMetaVar(self)
+
+ def mutate_over(self, visitor):
+ return visitor.visit_RevDBMetaVar(self)
+
+ def to_object(self, space):
+ w_node = space.call_function(get(space).w_RevDBMetaVar)
+ w_metavar = space.newint(self.metavar) # int
+ space.setattr(w_node, space.newtext('metavar'), w_metavar)
+ w_lineno = space.newint(self.lineno) # int
+ space.setattr(w_node, space.newtext('lineno'), w_lineno)
+ w_col_offset = space.newint(self.col_offset) # int
+ space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
+ return w_node
+
+ @staticmethod
+ def from_object(space, w_node):
+ w_metavar = get_field(space, w_node, 'metavar', False)
+ w_lineno = get_field(space, w_node, 'lineno', False)
+ w_col_offset = get_field(space, w_node, 'col_offset', False)
+ _metavar = space.int_w(w_metavar)
+ _lineno = space.int_w(w_lineno)
+ _col_offset = space.int_w(w_col_offset)
+ return RevDBMetaVar(_metavar, _lineno, _col_offset)
+
+State.ast_type('RevDBMetaVar', 'expr', ['metavar'])
+
+
class Attribute(expr):
def __init__(self, value, attr, ctx, lineno, col_offset):
@@ -3588,6 +3625,8 @@
return self.default_visitor(node)
def visit_Str(self, node):
return self.default_visitor(node)
+ def visit_RevDBMetaVar(self, node):
+ return self.default_visitor(node)
def visit_Attribute(self, node):
return self.default_visitor(node)
def visit_Subscript(self, node):
@@ -3804,6 +3843,9 @@
def visit_Str(self, node):
pass
+ def visit_RevDBMetaVar(self, node):
+ pass
+
def visit_Attribute(self, node):
node.value.walkabout(self)
diff --git a/pypy/interpreter/astcompiler/astbuilder.py
b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1165,6 +1165,11 @@
elif first_child_type == tokens.BACKQUOTE:
expr = self.handle_testlist(atom_node.get_child(1))
return ast.Repr(expr, atom_node.get_lineno(),
atom_node.get_column())
+ elif first_child_type == tokens.REVDBMETAVAR:
+ string = atom_node.get_child(0).get_value()
+ return ast.RevDBMetaVar(int(string[1:]),
+ atom_node.get_lineno(),
+ atom_node.get_column())
else:
raise AssertionError("unknown atom")
diff --git a/pypy/interpreter/astcompiler/codegen.py
b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1203,6 +1203,20 @@
sub.value.walkabout(self)
self._compile_slice(sub.slice, sub.ctx)
+ def _revdb_metavar(self, node):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import dbstate
+ if not dbstate.standard_code:
+ self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar)
+ return True
+ return False
+
+ def visit_RevDBMetaVar(self, node):
+ if self.space.reverse_debugging and self._revdb_metavar(node):
+ return
+ self.error("Unknown character ('$NUM' is only valid in the "
+ "reverse-debugger)", node)
+
class TopLevelCodeGenerator(PythonCodeGenerator):
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py
b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1096,7 +1096,7 @@
s = self.get_first_expr("'hi' ' implicitly' ' extra'")
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
- sentence = u"Die Männer ärgen sich!"
+ sentence = u"Die Männer ärgern sich!"
source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,)
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parser.parse_source(source.encode("utf-7"), info)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py
b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -27,7 +27,7 @@
generator._resolve_block_targets(blocks)
return generator, blocks
-class TestCompiler:
+class BaseTestCompiler:
"""These tests compile snippets of code and check them by
running them with our own interpreter. These are thus not
completely *unit* tests, but given that our interpreter is
@@ -74,6 +74,9 @@
def error_test(self, source, exc_type):
py.test.raises(exc_type, self.simple_test, source, None, None)
+
+class TestCompiler(BaseTestCompiler):
+
def test_issue_713(self):
func = "def f(_=2): return (_ if _ else _) if False else _"
yield self.st, func, "f()", 2
@@ -953,6 +956,22 @@
yield (self.st, "x=(lambda: (-0.0, 0.0), lambda: (0.0, -0.0))[1]()",
'repr(x)', '(0.0, -0.0)')
+class TestCompilerRevDB(BaseTestCompiler):
+ spaceconfig = {"translation.reverse_debugger": True}
+
+ def test_revdb_metavar(self):
+ from pypy.interpreter.reverse_debugging import dbstate, setup_revdb
+ self.space.reverse_debugging = True
+ try:
+ setup_revdb(self.space)
+ dbstate.standard_code = False
+ dbstate.metavars = [self.space.wrap(6)]
+ self.simple_test("x = 7*$0", "x", 42)
+ dbstate.standard_code = True
+ self.error_test("x = 7*$0", SyntaxError)
+ finally:
+ self.space.reverse_debugging = False
+
class AppTestCompiler:
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl
b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -71,6 +71,7 @@
| Repr(expr value)
| Num(object n) -- a number as a PyObject.
| Str(string s) -- need to specify raw, unicode, etc?
+ | RevDBMetaVar(int metavar)
-- other literals? bools?
-- the following expression can appear in assignment context
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -430,6 +430,8 @@
"""Base class for the interpreter-level implementations of object spaces.
http://pypy.readthedocs.org/en/latest/objspace.html""";
+ reverse_debugging = False
+
@not_rpython
def __init__(self, config=None):
"Basic initialization of objects."
@@ -441,6 +443,7 @@
from pypy.config.pypyoption import get_pypy_config
config = get_pypy_config(translating=False)
self.config = config
+ self.reverse_debugging = config.translation.reverse_debugger
self.builtin_modules = {}
self.reloading_modules = {}
@@ -458,6 +461,9 @@
def startup(self):
# To be called before using the space
+ if self.reverse_debugging:
+ self._revdb_startup()
+
self.threadlocals.enter_thread(self)
# Initialize already imported builtin modules
@@ -868,7 +874,8 @@
w_s1 = self.interned_strings.get(s)
if w_s1 is None:
w_s1 = w_s
- self.interned_strings.set(s, w_s1)
+ if self._side_effects_ok():
+ self.interned_strings.set(s, w_s1)
return w_s1
def new_interned_str(self, s):
@@ -878,9 +885,39 @@
w_s1 = self.interned_strings.get(s)
if w_s1 is None:
w_s1 = self.newtext(s)
- self.interned_strings.set(s, w_s1)
+ if self._side_effects_ok():
+ self.interned_strings.set(s, w_s1)
return w_s1
+ def _revdb_startup(self):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import setup_revdb
+ setup_revdb(self)
+
+ def _revdb_standard_code(self):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import dbstate
+ return dbstate.standard_code
+
+ def _side_effects_ok(self):
+ # For the reverse debugger: we run compiled watchpoint
+ # expressions in a fast way that will crash if they have
+ # side-effects. The obvious Python code with side-effects is
+ # documented "don't do that"; but some non-obvious side
+ # effects are also common, like interning strings (from
+ # unmarshalling the code object containing the watchpoint
+ # expression) to the two attribute caches in mapdict.py and
+ # typeobject.py. For now, we have to identify such places
+ # that are not acceptable for "reasonable" read-only
+ # watchpoint expressions, and write:
+ #
+ # if not space._side_effects_ok():
+ # don't cache.
+ #
+ if self.reverse_debugging:
+ return self._revdb_standard_code()
+ return True
+
def is_interned_str(self, s):
# interface for marshal_impl
if not we_are_translated():
diff --git a/pypy/interpreter/executioncontext.py
b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -64,6 +64,8 @@
return frame
def enter(self, frame):
+ if self.space.reverse_debugging:
+ self._revdb_enter(frame)
frame.f_backref = self.topframeref
self.topframeref = jit.virtual_ref(frame)
@@ -84,6 +86,8 @@
# be accessed also later
frame_vref()
jit.virtual_ref_finish(frame_vref, frame)
+ if self.space.reverse_debugging:
+ self._revdb_leave(got_exception)
# ________________________________________________________________
@@ -153,6 +157,8 @@
Like bytecode_trace() but doesn't invoke any other events besides the
trace function.
"""
+ if self.space.reverse_debugging:
+ self._revdb_potential_stop_point(frame)
if (frame.get_w_f_trace() is None or self.is_tracing or
self.gettrace() is None):
return
@@ -385,6 +391,21 @@
if self.space.check_signal_action is not None:
self.space.check_signal_action.perform(self, None)
+ def _revdb_enter(self, frame):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import enter_call
+ enter_call(self.topframeref(), frame)
+
+ def _revdb_leave(self, got_exception):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import leave_call
+ leave_call(self.topframeref(), got_exception)
+
+ def _revdb_potential_stop_point(self, frame):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import potential_stop_point
+ potential_stop_point(frame)
+
def _freeze_(self):
raise Exception("ExecutionContext instances should not be seen during"
" translation. Now is a good time to inspect the"
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -832,9 +832,11 @@
def fget_f_builtins(self, space):
return self.get_builtin().getdict(space)
+ def get_f_back(self):
+ return ExecutionContext.getnextframe_nohidden(self)
+
def fget_f_back(self, space):
- f_back = ExecutionContext.getnextframe_nohidden(self)
- return f_back
+ return self.get_f_back()
def fget_f_lasti(self, space):
return self.space.newint(self.last_instr)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -437,6 +437,8 @@
self.WITH_CLEANUP(oparg, next_instr)
elif opcode == opcodedesc.YIELD_VALUE.index:
self.YIELD_VALUE(oparg, next_instr)
+ elif opcode == opcodedesc.LOAD_REVDB_VAR.index:
+ self.LOAD_REVDB_VAR(oparg, next_instr)
else:
self.MISSING_OPCODE(oparg, next_instr)
@@ -1047,9 +1049,16 @@
def YIELD_VALUE(self, oparg, next_instr):
raise Yield
+ def _revdb_jump_backward(self, jumpto):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import jump_backward
+ jump_backward(self, jumpto)
+
def jump_absolute(self, jumpto, ec):
# this function is overridden by pypy.module.pypyjit.interp_jit
check_nonneg(jumpto)
+ if self.space.reverse_debugging:
+ self._revdb_jump_backward(jumpto)
return jumpto
def JUMP_FORWARD(self, jumpby, next_instr):
@@ -1303,6 +1312,18 @@
w_dict = self.peekvalue()
self.space.setitem(w_dict, w_key, w_value)
+ def _revdb_load_var(self, oparg):
+ # moved in its own function for the import statement
+ from pypy.interpreter.reverse_debugging import load_metavar
+ w_var = load_metavar(oparg)
+ self.pushvalue(w_var)
+
+ def LOAD_REVDB_VAR(self, oparg, next_instr):
+ if self.space.reverse_debugging:
+ self._revdb_load_var(oparg)
+ else:
+ self.MISSING_OPCODE(oparg, next_instr)
+
### ____________________________________________________________ ###
@@ -1607,7 +1628,7 @@
else:
skip_leading_underscores = False
for name in all:
- if skip_leading_underscores and name[0]=='_':
+ if skip_leading_underscores and name and name[0] == '_':
continue
into_locals[name] = getattr(module, name)
''', filename=__file__)
diff --git a/pypy/interpreter/pyparser/automata.py
b/pypy/interpreter/pyparser/automata.py
--- a/pypy/interpreter/pyparser/automata.py
+++ b/pypy/interpreter/pyparser/automata.py
@@ -23,6 +23,10 @@
ERROR_STATE = chr(255)
+# NB: all non-ascii bytes (>= 128) will be turned into 128
+NON_ASCII = chr(128)
+
+
class DFA:
# ____________________________________________________________
def __init__(self, states, accepts, start = 0):
@@ -36,7 +40,10 @@
for key in state:
if key == DEFAULT:
continue
- maximum = max(ord(key), maximum)
+ ordkey = ord(key)
+ if ordkey > 128:
+ raise ValueError("DFA does not support matching of
specific non-ASCII character %r. Use NON_ASCII instead" % key)
+ maximum = max(ordkey, maximum)
self.max_char = maximum + 1
defaults = []
@@ -72,6 +79,8 @@
i = pos
for i in range(pos, len(inVec)):
item = inVec[i]
+ if ord(item) > 0x80:
+ item = NON_ASCII
accept = self.accepts[crntState]
crntState = self._next_state(item, crntState)
if crntState != ERROR_STATE:
@@ -103,6 +112,8 @@
i = pos
for i in range(pos, len(inVec)):
item = inVec[i]
+ if ord(item) > 0x80:
+ item = NON_ASCII
accept = self.accepts[crntState]
if accept:
return i
diff --git a/pypy/interpreter/pyparser/data/Grammar2.7
b/pypy/interpreter/pyparser/data/Grammar2.7
--- a/pypy/interpreter/pyparser/data/Grammar2.7
+++ b/pypy/interpreter/pyparser/data/Grammar2.7
@@ -104,7 +104,7 @@
'[' [listmaker] ']' |
'{' [dictorsetmaker] '}' |
'`' testlist1 '`' |
- NAME | NUMBER | STRING+)
+ NAME | NUMBER | STRING+ | '$NUM')
listmaker: test ( list_for | (',' test)* [','] )
testlist_comp: test ( comp_for | (',' test)* [','] )
lambdef: 'lambda' [varargslist] ':' test
diff --git a/pypy/interpreter/pyparser/dfa_generated.py
b/pypy/interpreter/pyparser/dfa_generated.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -0,0 +1,299 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py
+# DO NOT EDIT
+# TO REGENERATE THE FILE, RUN:
+# python gendfa.py > dfa_generated.py
+
+from pypy.interpreter.pyparser import automata
+accepts = [True, True, True, True, True, True, True, True,
+ True, True, False, True, True, True, True, False,
+ False, False, False, True, False, False, True,
+ False, False, True, False, True, True, False,
+ True, False, False, True, False, False, True,
+ True, True, False, False, True, False, False,
+ False, True]
+states = [
+ # 0
+ {'\t': 0, '\n': 13, '\x0c': 0,
+ '\r': 14, ' ': 0, '!': 10, '"': 17,
+ '#': 19, '$': 15, '%': 12, '&': 12,
+ "'": 16, '(': 13, ')': 13, '*': 7,
+ '+': 12, ',': 13, '-': 12, '.': 6,
+ '/': 11, '0': 4, '1': 5, '2': 5,
+ '3': 5, '4': 5, '5': 5, '6': 5,
+ '7': 5, '8': 5, '9': 5, ':': 13,
+ ';': 13, '<': 9, '=': 12, '>': 8,
+ '@': 13, 'A': 1, 'B': 2, 'C': 1,
+ 'D': 1, 'E': 1, 'F': 1, 'G': 1,
+ 'H': 1, 'I': 1, 'J': 1, 'K': 1,
+ 'L': 1, 'M': 1, 'N': 1, 'O': 1,
+ 'P': 1, 'Q': 1, 'R': 3, 'S': 1,
+ 'T': 1, 'U': 2, 'V': 1, 'W': 1,
+ 'X': 1, 'Y': 1, 'Z': 1, '[': 13,
+ '\\': 18, ']': 13, '^': 12, '_': 1,
+ '`': 13, 'a': 1, 'b': 2, 'c': 1,
+ 'd': 1, 'e': 1, 'f': 1, 'g': 1,
+ 'h': 1, 'i': 1, 'j': 1, 'k': 1,
+ 'l': 1, 'm': 1, 'n': 1, 'o': 1,
+ 'p': 1, 'q': 1, 'r': 3, 's': 1,
+ 't': 1, 'u': 2, 'v': 1, 'w': 1,
+ 'x': 1, 'y': 1, 'z': 1, '{': 13,
+ '|': 12, '}': 13, '~': 13},
+ # 1
+ {'0': 1, '1': 1, '2': 1, '3': 1,
+ '4': 1, '5': 1, '6': 1, '7': 1,
+ '8': 1, '9': 1, 'A': 1, 'B': 1,
+ 'C': 1, 'D': 1, 'E': 1, 'F': 1,
+ 'G': 1, 'H': 1, 'I': 1, 'J': 1,
+ 'K': 1, 'L': 1, 'M': 1, 'N': 1,
+ 'O': 1, 'P': 1, 'Q': 1, 'R': 1,
+ 'S': 1, 'T': 1, 'U': 1, 'V': 1,
+ 'W': 1, 'X': 1, 'Y': 1, 'Z': 1,
+ '_': 1, 'a': 1, 'b': 1, 'c': 1,
+ 'd': 1, 'e': 1, 'f': 1, 'g': 1,
+ 'h': 1, 'i': 1, 'j': 1, 'k': 1,
+ 'l': 1, 'm': 1, 'n': 1, 'o': 1,
+ 'p': 1, 'q': 1, 'r': 1, 's': 1,
+ 't': 1, 'u': 1, 'v': 1, 'w': 1,
+ 'x': 1, 'y': 1, 'z': 1},
+ # 2
+ {'"': 17, "'": 16, '0': 1, '1': 1,
+ '2': 1, '3': 1, '4': 1, '5': 1,
+ '6': 1, '7': 1, '8': 1, '9': 1,
+ 'A': 1, 'B': 1, 'C': 1, 'D': 1,
+ 'E': 1, 'F': 1, 'G': 1, 'H': 1,
+ 'I': 1, 'J': 1, 'K': 1, 'L': 1,
+ 'M': 1, 'N': 1, 'O': 1, 'P': 1,
+ 'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+ 'U': 1, 'V': 1, 'W': 1, 'X': 1,
+ 'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+ 'b': 1, 'c': 1, 'd': 1, 'e': 1,
+ 'f': 1, 'g': 1, 'h': 1, 'i': 1,
+ 'j': 1, 'k': 1, 'l': 1, 'm': 1,
+ 'n': 1, 'o': 1, 'p': 1, 'q': 1,
+ 'r': 3, 's': 1, 't': 1, 'u': 1,
+ 'v': 1, 'w': 1, 'x': 1, 'y': 1,
+ 'z': 1},
+ # 3
+ {'"': 17, "'": 16, '0': 1, '1': 1,
+ '2': 1, '3': 1, '4': 1, '5': 1,
+ '6': 1, '7': 1, '8': 1, '9': 1,
+ 'A': 1, 'B': 1, 'C': 1, 'D': 1,
+ 'E': 1, 'F': 1, 'G': 1, 'H': 1,
+ 'I': 1, 'J': 1, 'K': 1, 'L': 1,
+ 'M': 1, 'N': 1, 'O': 1, 'P': 1,
+ 'Q': 1, 'R': 1, 'S': 1, 'T': 1,
+ 'U': 1, 'V': 1, 'W': 1, 'X': 1,
+ 'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+ 'b': 1, 'c': 1, 'd': 1, 'e': 1,
+ 'f': 1, 'g': 1, 'h': 1, 'i': 1,
+ 'j': 1, 'k': 1, 'l': 1, 'm': 1,
+ 'n': 1, 'o': 1, 'p': 1, 'q': 1,
+ 'r': 1, 's': 1, 't': 1, 'u': 1,
+ 'v': 1, 'w': 1, 'x': 1, 'y': 1,
+ 'z': 1},
+ # 4
+ {'.': 25, '0': 22, '1': 22, '2': 22,
+ '3': 22, '4': 22, '5': 22, '6': 22,
+ '7': 22, '8': 24, '9': 24, 'B': 23,
+ 'E': 26, 'J': 13, 'L': 13, 'O': 21,
+ 'X': 20, 'b': 23, 'e': 26, 'j': 13,
+ 'l': 13, 'o': 21, 'x': 20},
+ # 5
+ {'.': 25, '0': 5, '1': 5, '2': 5,
+ '3': 5, '4': 5, '5': 5, '6': 5,
+ '7': 5, '8': 5, '9': 5, 'E': 26,
+ 'J': 13, 'L': 13, 'e': 26, 'j': 13,
+ 'l': 13},
+ # 6
+ {'0': 27, '1': 27, '2': 27, '3': 27,
+ '4': 27, '5': 27, '6': 27, '7': 27,
+ '8': 27, '9': 27},
+ # 7
+ {'*': 12, '=': 13},
+ # 8
+ {'=': 13, '>': 12},
+ # 9
+ {'<': 12, '=': 13, '>': 13},
+ # 10
+ {'=': 13},
+ # 11
+ {'/': 12, '=': 13},
+ # 12
+ {'=': 13},
+ # 13
+ {},
+ # 14
+ {'\n': 13},
+ # 15
+ {'0': 28, '1': 28, '2': 28, '3': 28,
+ '4': 28, '5': 28, '6': 28, '7': 28,
+ '8': 28, '9': 28},
+ # 16
+ {automata.DEFAULT: 32, '\n': 29,
+ '\r': 29, "'": 30, '\\': 31},
+ # 17
+ {automata.DEFAULT: 35, '\n': 29,
+ '\r': 29, '"': 33, '\\': 34},
+ # 18
+ {'\n': 13, '\r': 14},
+ # 19
+ {automata.DEFAULT: 19, '\n': 29, '\r': 29},
+ # 20
+ {'0': 36, '1': 36, '2': 36, '3': 36,
+ '4': 36, '5': 36, '6': 36, '7': 36,
+ '8': 36, '9': 36, 'A': 36, 'B': 36,
+ 'C': 36, 'D': 36, 'E': 36, 'F': 36,
+ 'a': 36, 'b': 36, 'c': 36, 'd': 36,
+ 'e': 36, 'f': 36},
+ # 21
+ {'0': 37, '1': 37, '2': 37, '3': 37,
+ '4': 37, '5': 37, '6': 37, '7': 37},
+ # 22
+ {'.': 25, '0': 22, '1': 22, '2': 22,
+ '3': 22, '4': 22, '5': 22, '6': 22,
+ '7': 22, '8': 24, '9': 24, 'E': 26,
+ 'J': 13, 'L': 13, 'e': 26, 'j': 13,
+ 'l': 13},
+ # 23
+ {'0': 38, '1': 38},
+ # 24
+ {'.': 25, '0': 24, '1': 24, '2': 24,
+ '3': 24, '4': 24, '5': 24, '6': 24,
+ '7': 24, '8': 24, '9': 24, 'E': 26,
+ 'J': 13, 'e': 26, 'j': 13},
+ # 25
+ {'0': 25, '1': 25, '2': 25, '3': 25,
+ '4': 25, '5': 25, '6': 25, '7': 25,
+ '8': 25, '9': 25, 'E': 39, 'J': 13,
+ 'e': 39, 'j': 13},
+ # 26
+ {'+': 40, '-': 40, '0': 41, '1': 41,
+ '2': 41, '3': 41, '4': 41, '5': 41,
+ '6': 41, '7': 41, '8': 41, '9': 41},
+ # 27
+ {'0': 27, '1': 27, '2': 27, '3': 27,
+ '4': 27, '5': 27, '6': 27, '7': 27,
+ '8': 27, '9': 27, 'E': 39, 'J': 13,
+ 'e': 39, 'j': 13},
+ # 28
+ {'0': 28, '1': 28, '2': 28, '3': 28,
+ '4': 28, '5': 28, '6': 28, '7': 28,
+ '8': 28, '9': 28},
+ # 29
+ {},
+ # 30
+ {"'": 13},
+ # 31
+ {automata.DEFAULT: 42, '\n': 13, '\r': 14},
+ # 32
+ {automata.DEFAULT: 32, '\n': 29,
+ '\r': 29, "'": 13, '\\': 31},
+ # 33
+ {'"': 13},
+ # 34
+ {automata.DEFAULT: 43, '\n': 13, '\r': 14},
+ # 35
+ {automata.DEFAULT: 35, '\n': 29,
+ '\r': 29, '"': 13, '\\': 34},
+ # 36
+ {'0': 36, '1': 36, '2': 36, '3': 36,
+ '4': 36, '5': 36, '6': 36, '7': 36,
+ '8': 36, '9': 36, 'A': 36, 'B': 36,
+ 'C': 36, 'D': 36, 'E': 36, 'F': 36,
+ 'L': 13, 'a': 36, 'b': 36, 'c': 36,
+ 'd': 36, 'e': 36, 'f': 36, 'l': 13},
+ # 37
+ {'0': 37, '1': 37, '2': 37, '3': 37,
+ '4': 37, '5': 37, '6': 37, '7': 37,
+ 'L': 13, 'l': 13},
+ # 38
+ {'0': 38, '1': 38, 'L': 13, 'l': 13},
+ # 39
+ {'+': 44, '-': 44, '0': 45, '1': 45,
+ '2': 45, '3': 45, '4': 45, '5': 45,
+ '6': 45, '7': 45, '8': 45, '9': 45},
+ # 40
+ {'0': 41, '1': 41, '2': 41, '3': 41,
+ '4': 41, '5': 41, '6': 41, '7': 41,
+ '8': 41, '9': 41},
+ # 41
+ {'0': 41, '1': 41, '2': 41, '3': 41,
+ '4': 41, '5': 41, '6': 41, '7': 41,
+ '8': 41, '9': 41, 'J': 13, 'j': 13},
+ # 42
+ {automata.DEFAULT: 42, '\n': 29,
+ '\r': 29, "'": 13, '\\': 31},
+ # 43
+ {automata.DEFAULT: 43, '\n': 29,
+ '\r': 29, '"': 13, '\\': 34},
+ # 44
+ {'0': 45, '1': 45, '2': 45, '3': 45,
+ '4': 45, '5': 45, '6': 45, '7': 45,
+ '8': 45, '9': 45},
+ # 45
+ {'0': 45, '1': 45, '2': 45, '3': 45,
+ '4': 45, '5': 45, '6': 45, '7': 45,
+ '8': 45, '9': 45, 'J': 13, 'j': 13},
+ ]
+pseudoDFA = automata.DFA(states, accepts)
+
+accepts = [False, False, False, False, False, True]
+states = [
+ # 0
+ {automata.DEFAULT: 0, '"': 1, '\\': 2},
+ # 1
+ {automata.DEFAULT: 4, '"': 3, '\\': 2},
+ # 2
+ {automata.DEFAULT: 4},
+ # 3
+ {automata.DEFAULT: 4, '"': 5, '\\': 2},
+ # 4
+ {automata.DEFAULT: 4, '"': 1, '\\': 2},
+ # 5
+ {automata.DEFAULT: 4, '"': 5, '\\': 2},
+ ]
+double3DFA = automata.NonGreedyDFA(states, accepts)
+
+accepts = [False, False, False, False, False, True]
+states = [
+ # 0
+ {automata.DEFAULT: 0, "'": 1, '\\': 2},
+ # 1
+ {automata.DEFAULT: 4, "'": 3, '\\': 2},
+ # 2
+ {automata.DEFAULT: 4},
+ # 3
+ {automata.DEFAULT: 4, "'": 5, '\\': 2},
+ # 4
+ {automata.DEFAULT: 4, "'": 1, '\\': 2},
+ # 5
+ {automata.DEFAULT: 4, "'": 5, '\\': 2},
+ ]
+single3DFA = automata.NonGreedyDFA(states, accepts)
+
+accepts = [False, True, False, False]
+states = [
+ # 0
+ {automata.DEFAULT: 0, "'": 1, '\\': 2},
+ # 1
+ {},
+ # 2
+ {automata.DEFAULT: 3},
+ # 3
+ {automata.DEFAULT: 3, "'": 1, '\\': 2},
+ ]
+singleDFA = automata.DFA(states, accepts)
+
+accepts = [False, True, False, False]
+states = [
+ # 0
+ {automata.DEFAULT: 0, '"': 1, '\\': 2},
+ # 1
+ {},
+ # 2
+ {automata.DEFAULT: 3},
+ # 3
+ {automata.DEFAULT: 3, '"': 1, '\\': 2},
+ ]
+doubleDFA = automata.DFA(states, accepts)
+
diff --git a/pypy/interpreter/pyparser/future.py
b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -43,7 +43,7 @@
self.tok = self.tokens[index]
def skip(self, n):
- if self.tok[0] == n:
+ if self.tok.token_type == n:
self.next()
return True
else:
@@ -51,7 +51,7 @@
def skip_name(self, name):
from pypy.interpreter.pyparser import pygram
- if self.tok[0] == pygram.tokens.NAME and self.tok[1] == name:
+ if self.tok.token_type == pygram.tokens.NAME and self.tok.value ==
name:
self.next()
return True
else:
@@ -59,8 +59,8 @@
def next_feature_name(self):
from pypy.interpreter.pyparser import pygram
- if self.tok[0] == pygram.tokens.NAME:
- name = self.tok[1]
+ if self.tok.token_type == pygram.tokens.NAME:
+ name = self.tok.value
self.next()
if self.skip_name("as"):
self.skip(pygram.tokens.NAME)
@@ -101,7 +101,7 @@
# somewhere inside the last __future__ import statement
# (at the start would be fine too, but it's easier to grab a
# random position inside)
- last_position = (it.tok[2], it.tok[3])
+ last_position = (it.tok.lineno, it.tok.column)
result |= future_flags.get_compiler_feature(it.next_feature_name())
while it.skip(pygram.tokens.COMMA):
result |= future_flags.get_compiler_feature(it.next_feature_name())
diff --git a/pypy/interpreter/pyparser/gendfa.py
b/pypy/interpreter/pyparser/gendfa.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -0,0 +1,353 @@
+#! /usr/bin/env python
+"""Module gendfa
+
+Generates finite state automata for recognizing Python tokens. These are hand
+coded versions of the regular expressions originally appearing in Ping's
+tokenize module in the Python standard library.
+
+When run from the command line, this should pretty print the DFA machinery.
+
+To regenerate the dfa, run::
+
+ $ python gendfa.py > dfa_generated.py
+
+$Id: genPytokenize.py,v 1.1 2003/10/02 17:37:17 jriehl Exp $
+"""
+
+from pypy.interpreter.pyparser.pylexer import *
+from pypy.interpreter.pyparser.automata import NonGreedyDFA, DFA, DEFAULT
+
+def makePyPseudoDFA ():
+ import string
+ states = []
+ def makeEOL():
+ return group(states,
+ newArcPair(states, "\n"),
+ chain(states,
+ newArcPair(states, "\r"),
+ maybe(states, newArcPair(states, "\n"))))
+ # ____________________________________________________________
+ def makeLineCont ():
+ return chain(states,
+ newArcPair(states, "\\"),
+ makeEOL())
+ # ____________________________________________________________
+ # Ignore stuff
+ def makeWhitespace ():
+ return any(states, groupStr(states, " \f\t"))
+ # ____________________________________________________________
+ def makeComment ():
+ return chain(states,
+ newArcPair(states, "#"),
+ any(states, notGroupStr(states, "\r\n")))
+ # ____________________________________________________________
+ #ignore = chain(states,
+ # makeWhitespace(),
+ # any(states, chain(states,
+ # makeLineCont(),
+ # makeWhitespace())),
+ # maybe(states, makeComment()))
+ # ____________________________________________________________
+ # Names
+ name = chain(states,
+ groupStr(states, string.letters + "_"),
+ any(states, groupStr(states,
+ string.letters + string.digits + "_")))
+ # ____________________________________________________________
+ # Digits
+ def makeDigits ():
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
