Author: Ronan Lamy <ronan.l...@gmail.com>
Branch: multiphase
Changeset: r92111:474e84c737d0
Date: 2017-08-08 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/474e84c737d0/
Log: hg merge py3.5
diff too long, truncating to 2000 out of 23586 lines
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -27,16 +27,17 @@
^pypy/module/cpyext/test/.+\.manifest$
^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
-^pypy/module/cppyy/src/.+\.o$
-^pypy/module/cppyy/bench/.+\.so$
-^pypy/module/cppyy/bench/.+\.root$
-^pypy/module/cppyy/bench/.+\.d$
-^pypy/module/cppyy/src/.+\.errors$
-^pypy/module/cppyy/test/.+_rflx\.cpp$
-^pypy/module/cppyy/test/.+\.so$
-^pypy/module/cppyy/test/.+\.rootmap$
-^pypy/module/cppyy/test/.+\.exe$
-^pypy/module/cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/src/.+\.o$
+^pypy/module/_cppyy/bench/.+\.so$
+^pypy/module/_cppyy/bench/.+\.root$
+^pypy/module/_cppyy/bench/.+\.d$
+^pypy/module/_cppyy/src/.+\.errors$
+^pypy/module/_cppyy/test/.+_rflx\.cpp$
+^pypy/module/_cppyy/test/.+\.so$
+^pypy/module/_cppyy/test/.+\.rootmap$
+^pypy/module/_cppyy/test/.+\.exe$
+^pypy/module/_cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/.+/*\.pcm$
^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$
^pypy/doc/.+\.html$
^pypy/doc/config/.+\.rst$
@@ -93,6 +94,3 @@
^release/
^rpython/_cache$
-pypy/module/cppyy/.+/*\.pcm
-
-
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@
RUNINTERP = $(PYPY_EXECUTABLE)
endif
-.PHONY: cffi_imports
+.PHONY: pypy-c cffi_imports
pypy-c:
@echo
@@ -32,7 +32,7 @@
@echo
"===================================================================="
@echo
@sleep 5
- $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+ cd pypy/goal && $(RUNINTERP) ../../rpython/bin/rpython -Ojit
targetpypystandalone.py
# Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are
# replaced with an opaque --jobserver option by the time this Makefile
@@ -40,4 +40,4 @@
# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
cffi_imports: pypy-c
- PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
+ PYTHONPATH=. pypy/goal/pypy-c pypy/tool/build_cffi_imports.py ||
/bin/true
diff --git a/lib-python/2.7/distutils/unixccompiler.py
b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -226,7 +226,19 @@
return "-L" + dir
def _is_gcc(self, compiler_name):
- return "gcc" in compiler_name or "g++" in compiler_name
+ # XXX PyPy workaround, look at the big comment below for more
+ # context. On CPython, the hack below works fine because
+ # `compiler_name` contains the name of the actual compiler which was
+ # used at compile time (e.g. 'x86_64-linux-gnu-gcc' on my machine).
+ # PyPy hardcodes it to 'cc', so the hack doesn't work, and the end
+ # result is that we pass the wrong option to the compiler.
+ #
+ # The workaround is to *always* pretend to be GCC if we are on Linux:
+ # this should cover the vast majority of real systems, including the
+ # ones which use clang (which understands the '-Wl,-rpath' syntax as
+ # well)
+ return (sys.platform == "linux2" or
+ "gcc" in compiler_name or "g++" in compiler_name)
def runtime_library_dir_option(self, dir):
# XXX Hackish, at the very least. See Python bug #445902:
diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py
--- a/lib-python/3/stat.py
+++ b/lib-python/3/stat.py
@@ -139,13 +139,21 @@
def filemode(mode):
"""Convert a file's mode to a string of the form '-rwxrwxrwx'."""
perm = []
+
+ # The first group gets a question mark if none of the bits match the mode.
+ empty = "?"
+
for table in _filemode_table:
for bit, char in table:
if mode & bit == bit:
perm.append(char)
break
else:
- perm.append("-")
+ perm.append(empty)
+
+ # All the rest of the positions get a - if the bits don't match.
+ empty = "-"
+
return "".join(perm)
diff --git a/lib-python/3/test/test_stat.py b/lib-python/3/test/test_stat.py
--- a/lib-python/3/test/test_stat.py
+++ b/lib-python/3/test/test_stat.py
@@ -138,6 +138,10 @@
self.assertS_IS("REG", st_mode)
self.assertEqual(modestr, '-r--r--r--')
self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444)
+
+ # If there are only permission bits, no type bytes, a question
+ # mark is rendered in the type field.
+ self.assertEqual(self.statmod.filemode(0o420), '?r---w----')
else:
os.chmod(TESTFN, 0o700)
st_mode, modestr = self.get_mode()
diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -173,14 +173,13 @@
return tuple(dn)
-STATIC_BIO_BUF = ffi.new("char[]", 2048)
-
def _bio_get_str(biobuf):
- length = lib.BIO_gets(biobuf, STATIC_BIO_BUF, len(STATIC_BIO_BUF)-1)
+ bio_buf = ffi.new("char[]", 2048)
+ length = lib.BIO_gets(biobuf, bio_buf, len(bio_buf)-1)
if length < 0:
if biobuf: lib.BIO_free(biobuf)
raise ssl_error(None)
- return _str_with_len(STATIC_BIO_BUF, length)
+ return _str_with_len(bio_buf, length)
def _decode_certificate(certificate):
retval = {}
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -411,7 +411,7 @@
val = lib.mvwget_wch(self._win, *args, wch)
else:
raise error("get_wch requires 0 or 2 arguments")
- _check_ERR(val, "get_wch"):
+ _check_ERR(val, "get_wch")
return wch[0]
def getkey(self, *args):
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -22,12 +22,27 @@
linklibs = ['tcl', 'tk']
libdirs = []
else:
- for _ver in ['', '8.6', '8.5', '']:
+ # On some Linux distributions, the tcl and tk libraries are
+ # stored in /usr/include, so we must check this case also
+ libdirs = []
+ found = False
+ for _ver in ['', '8.6', '8.5']:
incdirs = ['/usr/include/tcl' + _ver]
linklibs = ['tcl' + _ver, 'tk' + _ver]
- libdirs = []
if os.path.isdir(incdirs[0]):
+ found = True
break
+ if not found:
+ for _ver in ['8.6', '8.5', '']:
+ incdirs = []
+ linklibs = ['tcl' + _ver, 'tk' + _ver]
+ if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])):
+ found = True
+ break
+ if not found:
+ sys.stderr.write("*** TCL libraries not found! Falling back...\n")
+ incdirs = []
+ linklibs = ['tcl', 'tk']
config_ffi = FFI()
config_ffi.cdef("""
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -95,6 +95,7 @@
#define _cffi_from_c_ulong PyLong_FromUnsignedLong
#define _cffi_from_c_longlong PyLong_FromLongLong
#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
#define _cffi_to_c_double PyFloat_AsDouble
#define _cffi_to_c_float PyFloat_AsDouble
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -1,7 +1,12 @@
/***** Support code for embedding *****/
-#if defined(_MSC_VER)
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#if defined(_WIN32)
# define CFFI_DLLEXPORT __declspec(dllexport)
#elif defined(__GNUC__)
# define CFFI_DLLEXPORT __attribute__((visibility("default")))
@@ -525,3 +530,7 @@
#undef cffi_compare_and_swap
#undef cffi_write_barrier
#undef cffi_read_barrier
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -394,12 +394,17 @@
replace_with = ' ' + replace_with
return self._backend.getcname(cdecl, replace_with)
- def gc(self, cdata, destructor):
+ def gc(self, cdata, destructor, size=0):
"""Return a new cdata object that points to the same
data. Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called.
+
+ The optional 'size' gives an estimate of the size, used to
+ trigger the garbage collection more eagerly. So far only used
+ on PyPy. It tells the GC that the returned object keeps alive
+ roughly 'size' bytes of external memory.
"""
- return self._backend.gcp(cdata, destructor)
+ return self._backend.gcp(cdata, destructor, size)
def _get_cached_btype(self, type):
assert self._lock.acquire(False) is False
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1002,7 +1002,7 @@
_weakref_cache_ref = None
- def gcp(self, cdata, destructor):
+ def gcp(self, cdata, destructor, size=0):
if self._weakref_cache_ref is None:
import weakref
class MyRef(weakref.ref):
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -412,6 +412,9 @@
prnt(' }')
prnt(' p[0] = (const void *)0x%x;' % self._version)
prnt(' p[1] = &_cffi_type_context;')
+ prnt('#if PY_MAJOR_VERSION >= 3')
+ prnt(' return NULL;')
+ prnt('#endif')
prnt('}')
# on Windows, distutils insists on putting init_cffi_xyz in
# 'export_symbols', so instead of fighting it, just give up and
@@ -578,7 +581,7 @@
def _convert_expr_from_c(self, tp, var, context):
if isinstance(tp, model.BasePrimitiveType):
- if tp.is_integer_type():
+ if tp.is_integer_type() and tp.name != '_Bool':
return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
elif isinstance(tp, model.UnknownFloatType):
return '_cffi_from_c_double(%s)' % (var,)
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -296,7 +296,7 @@
def _convert_expr_from_c(self, tp, var, context):
if isinstance(tp, model.PrimitiveType):
- if tp.is_integer_type():
+ if tp.is_integer_type() and tp.name != '_Bool':
return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
elif tp.name != 'long double':
return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
@@ -872,6 +872,7 @@
#define _cffi_from_c_ulong PyLong_FromUnsignedLong
#define _cffi_from_c_longlong PyLong_FromLongLong
#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
#define _cffi_to_c_double PyFloat_AsDouble
#define _cffi_to_c_float PyFloat_AsDouble
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,7 +39,7 @@
"thread", "itertools", "pyexpat", "cpyext", "array",
"binascii", "_multiprocessing", '_warnings', "_collections",
"_multibytecodec", "_continuation", "_cffi_backend",
- "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
+ "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
"_jitlog",
])
@@ -71,8 +71,8 @@
if name in translation_modules:
translation_modules.remove(name)
- if "cppyy" in working_modules:
- working_modules.remove("cppyy") # not tested on win32
+ if "_cppyy" in working_modules:
+ working_modules.remove("_cppyy") # not tested on win32
# The _locale module is needed by site.py on Windows
default_modules.add("_locale")
@@ -81,8 +81,8 @@
working_modules.remove('fcntl') # LOCK_NB not defined
working_modules.remove("_minimal_curses")
working_modules.remove("termios")
- if "cppyy" in working_modules:
- working_modules.remove("cppyy") # depends on ctypes
+ if "_cppyy" in working_modules:
+ working_modules.remove("_cppyy") # depends on ctypes
#if sys.platform.startswith("linux"):
# _mach = os.popen('uname -m', 'r').read().strip()
@@ -94,7 +94,7 @@
'_multiprocessing': [('objspace.usemodules.time', True),
('objspace.usemodules.thread', True)],
'cpyext': [('objspace.usemodules.array', True)],
- 'cppyy': [('objspace.usemodules.cpyext', True)],
+ '_cppyy': [('objspace.usemodules.cpyext', True)],
'faulthandler': [('objspace.usemodules._vmprof', True)],
}
module_suggests = {
@@ -227,11 +227,6 @@
"use specialised tuples",
default=False),
- BoolOption("withcelldict",
- "use dictionaries that are optimized for being used as
module dicts",
- default=False,
- requires=[("objspace.honor__builtins__", False)]),
-
BoolOption("withliststrategies",
"enable optimized ways to store lists of primitives ",
default=True),
@@ -291,7 +286,7 @@
# extra optimizations with the JIT
if level == 'jit':
- config.objspace.std.suggest(withcelldict=True)
+ pass # none at the moment
def enable_allworkingmodules(config):
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -10,6 +10,18 @@
minutes on a fast machine -- and RAM-hungry. You will need **at least** 2 GB
of memory on a 32-bit machine and 4GB on a 64-bit machine.
+Before you start
+----------------
+
+Our normal development workflow avoids a full translation by using test-driven
+development. You can read more about how to develop PyPy here_, and latest
+translated (hopefully functional) binary packages are available on our
+buildbot's `nightly builds`_
+
+.. _here: getting-started-dev.html
+.. _`nightly builds`: http://buildbot.pypy.org/nightly
+
+You will need the build dependencies below to run the tests.
Clone the repository
--------------------
@@ -140,22 +152,61 @@
Run the translation
-------------------
+We usually translate in the ``pypy/goal`` directory, so all the following
+commands assume your ``$pwd`` is there.
+
Translate with JIT::
- cd pypy/goal
pypy ../../rpython/bin/rpython --opt=jit
Translate without JIT::
- cd pypy/goal
pypy ../../rpython/bin/rpython --opt=2
+Note this translates pypy via the ``targetpypystandalone.py`` file, so these
+are shorthand for::
+
+ pypy ../../rpython/bin/rpython <rpython options> targetpypystandalone.py
<pypy options>
+
+More help is availabe via ``--help`` at either option position, and more info
+can be found in the :doc:`config/index` section.
+
(You can use ``python`` instead of ``pypy`` here, which will take longer
but works too.)
-If everything works correctly this will create an executable ``pypy-c`` in the
-current directory. The executable behaves mostly like a normal Python
-interpreter (see :doc:`cpython_differences`).
+If everything works correctly this will:
+
+1. Run the rpython `translation chain`_, producing a database of the
+ entire pypy interpreter. This step is currently singe threaded, and RAM
+ hungry. As part of this step, the chain creates a large number of C code
+ files and a Makefile to compile them in a
+ directory controlled by the ``PYPY_USESSION_DIR`` environment variable.
+2. Create an executable ``pypy-c`` by running the Makefile. This step can
+ utilize all possible cores on the machine.
+3. Copy the needed binaries to the current directory.
+4. Generate c-extension modules for any cffi-based stdlib modules.
+
+
+The resulting executable behaves mostly like a normal Python
+interpreter (see :doc:`cpython_differences`), and is ready for testing, for
+use as a base interpreter for a new virtualenv, or for packaging into a binary
+suitable for installation on another machine running the same OS as the build
+machine.
+
+Note that step 4 is merely done as a convenience, any of the steps may be rerun
+without rerunning the previous steps.
+
+.. _`translation chain`:
https://rpython.readthedocs.io/en/latest/translation.html
+
+
+Making a debug build of PyPy
+----------------------------
+
+If the Makefile is rerun with the lldebug or lldebug0 target, appropriate
+compilation flags are added to add debug info and reduce compiler optimizations
+to ``-O0`` respectively. If you stop in a debugger, you will see the
+very wordy machine-generated C code from the rpython translation step, which
+takes a little bit of reading to relate back to the rpython code.
Build cffi import libraries for the stdlib
------------------------------------------
@@ -169,14 +220,6 @@
.. _`out-of-line API mode`:
http://cffi.readthedocs.org/en/latest/overview.html#real-example-api-level-out-of-line
-Translating with non-standard options
--------------------------------------
-
-It is possible to have non-standard features enabled for translation,
-but they are not really tested any more. Look, for example, at the
-:doc:`objspace proxies <objspace-proxies>` document.
-
-
Packaging (preparing for installation)
--------------------------------------
@@ -205,14 +248,16 @@
* PyPy 2.5.1 or earlier: normal users would see permission errors.
Installers need to run ``pypy -c "import gdbm"`` and other similar
- commands at install time; the exact list is in `package.py`_. Users
+ commands at install time; the exact list is in
+ :source:`pypy/tool/release/package.py <package.py>`. Users
seeing a broken installation of PyPy can fix it after-the-fact if they
have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
* PyPy 2.6 and later: anyone would get ``ImportError: no module named
_gdbm_cffi``. Installers need to run ``pypy _gdbm_build.py`` in the
``lib_pypy`` directory during the installation process (plus others;
- see the exact list in `package.py`_). Users seeing a broken
+ see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+ Users seeing a broken
installation of PyPy can fix it after-the-fact, by running ``pypy
/path/to/lib_pypy/_gdbm_build.py``. This command produces a file
called ``_gdbm_cffi.pypy-41.so`` locally, which is a C extension
diff --git a/pypy/doc/config/objspace.std.withcelldict.txt
b/pypy/doc/config/objspace.std.withcelldict.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.withcelldict.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Enable cell-dicts. This optimization is not helpful without the JIT. In the
-presence of the JIT, it greatly helps looking up globals.
diff --git a/pypy/doc/configuration.rst b/pypy/doc/configuration.rst
--- a/pypy/doc/configuration.rst
+++ b/pypy/doc/configuration.rst
@@ -188,4 +188,6 @@
can be found on the ``config`` attribute of all ``TranslationContext``
instances and are described in :source:`rpython/config/translationoption.py`.
The interpreter options
are attached to the object space, also under the name ``config`` and are
-described in :source:`pypy/config/pypyoption.py`.
+described in :source:`pypy/config/pypyoption.py`. Both set of options are
+documented in the :doc:`config/index` section.
+
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
deleted file mode 100644
--- a/pypy/doc/cppyy.rst
+++ /dev/null
@@ -1,672 +0,0 @@
-cppyy: C++ bindings for PyPy
-============================
-
-The cppyy module delivers dynamic Python-C++ bindings.
-It is designed for automation, high performance, scale, interactivity, and
-handling all of modern C++ (11, 14, etc.).
-It is based on `Cling`_ which, through `LLVM`_/`clang`_, provides C++
-reflection and interactivity.
-Reflection information is extracted from C++ header files.
-Cppyy itself is built into PyPy (an alternative exists for CPython), but
-it requires a `backend`_, installable through pip, to interface with Cling.
-
-.. _Cling: https://root.cern.ch/cling
-.. _LLVM: http://llvm.org/
-.. _clang: http://clang.llvm.org/
-.. _backend: https://pypi.python.org/pypi/PyPy-cppyy-backend
-
-
-Installation
-------------
-
-This assumes PyPy2.7 v5.7 or later; earlier versions use a Reflex-based cppyy
-module, which is no longer supported.
-Both the tooling and user-facing Python codes are very backwards compatible,
-however.
-Further dependencies are cmake (for general build), Python2.7 (for LLVM), and
-a modern C++ compiler (one that supports at least C++11).
-
-Assuming you have a recent enough version of PyPy installed, use pip to
-complete the installation of cppyy::
-
- $ MAKE_NPROCS=4 pypy-c -m pip install --verbose PyPy-cppyy-backend
-
-Set the number of parallel builds ('4' in this example, through the MAKE_NPROCS
-environment variable) to a number appropriate for your machine.
-The building process may take quite some time as it includes a customized
-version of LLVM as part of Cling, which is why --verbose is recommended so that
-you can see the build progress.
-
-The default installation will be under
-$PYTHONHOME/site-packages/cppyy_backend/lib,
-which needs to be added to your dynamic loader path (LD_LIBRARY_PATH).
-If you need the dictionary and class map generation tools (used in the examples
-below), you need to add $PYTHONHOME/site-packages/cppyy_backend/bin to your
-executable path (PATH).
-
-
-Basic bindings example
-----------------------
-
-These examples assume that cppyy_backend is pointed to by the environment
-variable CPPYYHOME, and that CPPYYHOME/lib is added to LD_LIBRARY_PATH and
-CPPYYHOME/bin to PATH.
-
-Let's first test with a trivial example whether all packages are properly
-installed and functional.
-Create a C++ header file with some class in it (all functions are made inline
-for convenience; if you have out-of-line code, link with it as appropriate)::
-
- $ cat MyClass.h
- class MyClass {
- public:
- MyClass(int i = -99) : m_myint(i) {}
-
- int GetMyInt() { return m_myint; }
- void SetMyInt(int i) { m_myint = i; }
-
- public:
- int m_myint;
- };
-
-Then, generate the bindings using ``genreflex`` (installed under
-cppyy_backend/bin in site_packages), and compile the code::
-
- $ genreflex MyClass.h
- $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include
MyClass_rflx.cpp -o libMyClassDict.so -L$CPPYYHOME/lib -lCling
-
-Next, make sure that the library can be found through the dynamic lookup path
-(the ``LD_LIBRARY_PATH`` environment variable on Linux, ``PATH`` on Windows),
-for example by adding ".".
-Now you're ready to use the bindings.
-Since the bindings are designed to look pythonistic, it should be
-straightforward::
-
- $ pypy-c
- >>>> import cppyy
- >>>> cppyy.load_reflection_info("libMyClassDict.so")
- <CPPLibrary object at 0xb6fd7c4c>
- >>>> myinst = cppyy.gbl.MyClass(42)
- >>>> print myinst.GetMyInt()
- 42
- >>>> myinst.SetMyInt(33)
- >>>> print myinst.m_myint
- 33
- >>>> myinst.m_myint = 77
- >>>> print myinst.GetMyInt()
- 77
- >>>> help(cppyy.gbl.MyClass) # shows that normal python introspection
works
-
-That's all there is to it!
-
-
-Automatic class loader
-----------------------
-
-There is one big problem in the code above, that prevents its use in a (large
-scale) production setting: the explicit loading of the reflection library.
-Clearly, if explicit load statements such as these show up in code downstream
-from the ``MyClass`` package, then that prevents the ``MyClass`` author from
-repackaging or even simply renaming the dictionary library.
-
-The solution is to make use of an automatic class loader, so that downstream
-code never has to call ``load_reflection_info()`` directly.
-The class loader makes use of so-called rootmap files, which ``genreflex``
-can produce.
-These files contain the list of available C++ classes and specify the library
-that needs to be loaded for their use (as an aside, this listing allows for a
-cross-check to see whether reflection info is generated for all classes that
-you expect).
-By convention, the rootmap files should be located next to the reflection info
-libraries, so that they can be found through the normal shared library search
-path.
-They can be concatenated together, or consist of a single rootmap file per
-library.
-For example::
-
- $ genreflex MyClass.h --rootmap=libMyClassDict.rootmap
--rootmap-lib=libMyClassDict.so
- $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include
MyClass_rflx.cpp -o libMyClassDict.so -L$CPPYYHOME/lib -lCling
-
-where the first option (``--rootmap``) specifies the output file name, and the
-second option (``--rootmap-lib``) the name of the reflection library where
-``MyClass`` will live.
-It is necessary to provide that name explicitly, since it is only in the
-separate linking step where this name is fixed.
-If the second option is not given, the library is assumed to be libMyClass.so,
-a name that is derived from the name of the header file.
-
-With the rootmap file in place, the above example can be rerun without explicit
-loading of the reflection info library::
-
- $ pypy-c
- >>>> import cppyy
- >>>> myinst = cppyy.gbl.MyClass(42)
- >>>> print myinst.GetMyInt()
- 42
- >>>> # etc. ...
-
-As a caveat, note that the class loader is currently limited to classes only.
-
-
-Advanced example
-----------------
-
-The following snippet of C++ is very contrived, to allow showing that such
-pathological code can be handled and to show how certain features play out in
-practice::
-
- $ cat MyAdvanced.h
- #include <string>
-
- class Base1 {
- public:
- Base1(int i) : m_i(i) {}
- virtual ~Base1() {}
- int m_i;
- };
-
- class Base2 {
- public:
- Base2(double d) : m_d(d) {}
- virtual ~Base2() {}
- double m_d;
- };
-
- class C;
-
- class Derived : public virtual Base1, public virtual Base2 {
- public:
- Derived(const std::string& name, int i, double d) : Base1(i),
Base2(d), m_name(name) {}
- virtual C* gimeC() { return (C*)0; }
- std::string m_name;
- };
-
- Base2* BaseFactory(const std::string& name, int i, double d) {
- return new Derived(name, i, d);
- }
-
-This code is still only in a header file, with all functions inline, for
-convenience of the example.
-If the implementations live in a separate source file or shared library, the
-only change needed is to link those in when building the reflection library.
-
-If you were to run ``genreflex`` like above in the basic example, you will
-find that not all classes of interest will be reflected, nor will be the
-global factory function.
-In particular, ``std::string`` will be missing, since it is not defined in
-this header file, but in a header file that is included.
-In practical terms, general classes such as ``std::string`` should live in a
-core reflection set, but for the moment assume we want to have it in the
-reflection library that we are building for this example.
-
-The ``genreflex`` script can be steered using a so-called `selection file`_
-(see "Generating Reflex Dictionaries")
-which is a simple XML file specifying, either explicitly or by using a
-pattern, which classes, variables, namespaces, etc. to select from the given
-header file.
-With the aid of a selection file, a large project can be easily managed:
-simply ``#include`` all relevant headers into a single header file that is
-handed to ``genreflex``.
-In fact, if you hand multiple header files to ``genreflex``, then a selection
-file is almost obligatory: without it, only classes from the last header will
-be selected.
-Then, apply a selection file to pick up all the relevant classes.
-For our purposes, the following rather straightforward selection will do
-(the name ``lcgdict`` for the root is historical, but required)::
-
- $ cat MyAdvanced.xml
- <lcgdict>
- <class pattern="Base?" />
- <class name="Derived" />
- <class name="std::string" />
- <function name="BaseFactory" />
- </lcgdict>
-
-.. _selection file: https://root.cern.ch/how/how-use-reflex
-
-Now the reflection info can be generated and compiled::
-
- $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
- $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include
MyAdvanced_rflx.cpp -o libAdvExDict.so -L$CPPYYHOME/lib -lCling
-
-and subsequently be used from PyPy::
-
- >>>> import cppyy
- >>>> cppyy.load_reflection_info("libAdvExDict.so")
- <CPPLibrary object at 0x00007fdb48fc8120>
- >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
- >>>> type(d)
- <class '__main__.Derived'>
- >>>> isinstance(d, cppyy.gbl.Base1)
- True
- >>>> isinstance(d, cppyy.gbl.Base2)
- True
- >>>> d.m_i, d.m_d
- (42, 3.14)
- >>>> d.m_name == "name"
- True
- >>>>
-
-Again, that's all there is to it!
-
-A couple of things to note, though.
-If you look back at the C++ definition of the ``BaseFactory`` function,
-you will see that it declares the return type to be a ``Base2``, yet the
-bindings return an object of the actual type ``Derived``?
-This choice is made for a couple of reasons.
-First, it makes method dispatching easier: if bound objects are always their
-most derived type, then it is easy to calculate any offsets, if necessary.
-Second, it makes memory management easier: the combination of the type and
-the memory address uniquely identifies an object.
-That way, it can be recycled and object identity can be maintained if it is
-entered as a function argument into C++ and comes back to PyPy as a return
-value.
-Last, but not least, casting is decidedly unpythonistic.
-By always providing the most derived type known, casting becomes unnecessary.
-For example, the data member of ``Base2`` is simply directly available.
-Note also that the unreflected ``gimeC`` method of ``Derived`` does not
-preclude its use.
-It is only the ``gimeC`` method that is unusable as long as class ``C`` is
-unknown to the system.
-
-
-Features
---------
-
-The following is not meant to be an exhaustive list, since cppyy is still
-under active development.
-Furthermore, the intention is that every feature is as natural as possible on
-the python side, so if you find something missing in the list below, simply
-try it out.
-It is not always possible to provide exact mapping between python and C++
-(active memory management is one such case), but by and large, if the use of a
-feature does not strike you as obvious, it is more likely to simply be a bug.
-That is a strong statement to make, but also a worthy goal.
-For the C++ side of the examples, refer to this :doc:`example code
<cppyy_example>`, which was
-bound using::
-
- $ genreflex example.h --deep --rootmap=libexampleDict.rootmap
--rootmap-lib=libexampleDict.so
- $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include
example_rflx.cpp -o libexampleDict.so -L$CPPYYHOME/lib -lCling
-
-* **abstract classes**: Are represented as python classes, since they are
- needed to complete the inheritance hierarchies, but will raise an exception
- if an attempt is made to instantiate from them.
- Example::
-
- >>>> from cppyy.gbl import AbstractClass, ConcreteClass
- >>>> a = AbstractClass()
- Traceback (most recent call last):
- File "<console>", line 1, in <module>
- TypeError: cannot instantiate abstract class 'AbstractClass'
- >>>> issubclass(ConcreteClass, AbstractClass)
- True
- >>>> c = ConcreteClass()
- >>>> isinstance(c, AbstractClass)
- True
- >>>>
-
-* **arrays**: Supported for builtin data types only, as used from module
- ``array``.
- Out-of-bounds checking is limited to those cases where the size is known at
- compile time (and hence part of the reflection info).
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass
- >>>> from array import array
- >>>> c = ConcreteClass()
- >>>> c.array_method(array('d', [1., 2., 3., 4.]), 4)
- 1 2 3 4
- >>>>
-
-* **builtin data types**: Map onto the expected equivalent python types, with
- the caveat that there may be size differences, and thus it is possible that
- exceptions are raised if an overflow is detected.
-
-* **casting**: Is supposed to be unnecessary.
- Object pointer returns from functions provide the most derived class known
- in the hierarchy of the object being returned.
- This is important to preserve object identity as well as to make casting,
- a pure C++ feature after all, superfluous.
- Example::
-
- >>>> from cppyy.gbl import AbstractClass, ConcreteClass
- >>>> c = ConcreteClass()
- >>>> ConcreteClass.show_autocast.__doc__
- 'AbstractClass* ConcreteClass::show_autocast()'
- >>>> d = c.show_autocast()
- >>>> type(d)
- <class '__main__.ConcreteClass'>
- >>>>
-
- However, if need be, you can perform C++-style reinterpret_casts (i.e.
- without taking offsets into account), by taking and rebinding the address
- of an object::
-
- >>>> from cppyy import addressof, bind_object
- >>>> e = bind_object(addressof(d), AbstractClass)
- >>>> type(e)
- <class '__main__.AbstractClass'>
- >>>>
-
-* **classes and structs**: Get mapped onto python classes, where they can be
- instantiated as expected.
- If classes are inner classes or live in a namespace, their naming and
- location will reflect that.
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass, Namespace
- >>>> ConcreteClass == Namespace.ConcreteClass
- False
- >>>> n = Namespace.ConcreteClass.NestedClass()
- >>>> type(n)
- <class '__main__.Namespace::ConcreteClass::NestedClass'>
- >>>>
-
-* **data members**: Public data members are represented as python properties
- and provide read and write access on instances as expected.
- Private and protected data members are not accessible.
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass
- >>>> c = ConcreteClass()
- >>>> c.m_int
- 42
- >>>>
-
-* **default arguments**: C++ default arguments work as expected, but python
- keywords are not supported.
- It is technically possible to support keywords, but for the C++ interface,
- the formal argument names have no meaning and are not considered part of the
- API, hence it is not a good idea to use keywords.
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass
- >>>> c = ConcreteClass() # uses default argument
- >>>> c.m_int
- 42
- >>>> c = ConcreteClass(13)
- >>>> c.m_int
- 13
- >>>>
-
-* **doc strings**: The doc string of a method or function contains the C++
- arguments and return types of all overloads of that name, as applicable.
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass
- >>>> print ConcreteClass.array_method.__doc__
- void ConcreteClass::array_method(int*, int)
- void ConcreteClass::array_method(double*, int)
- >>>>
-
-* **enums**: Are translated as ints with no further checking.
-
-* **functions**: Work as expected and live in their appropriate namespace
- (which can be the global one, ``cppyy.gbl``).
-
-* **inheritance**: All combinations of inheritance on the C++ (single,
- multiple, virtual) are supported in the binding.
- However, new python classes can only use single inheritance from a bound C++
- class.
- Multiple inheritance would introduce two "this" pointers in the binding.
- This is a current, not a fundamental, limitation.
- The C++ side will not see any overridden methods on the python side, as
- cross-inheritance is planned but not yet supported.
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass
- >>>> help(ConcreteClass)
- Help on class ConcreteClass in module __main__:
-
- class ConcreteClass(AbstractClass)
- | Method resolution order:
- | ConcreteClass
- | AbstractClass
- | cppyy.CPPObject
- | __builtin__.CPPInstance
- | __builtin__.object
- |
- | Methods defined here:
- |
- | ConcreteClass(self, *args)
- | ConcreteClass::ConcreteClass(const ConcreteClass&)
- | ConcreteClass::ConcreteClass(int)
- | ConcreteClass::ConcreteClass()
- |
- etc. ....
-
-* **memory**: C++ instances created by calling their constructor from python
- are owned by python.
- You can check/change the ownership with the _python_owns flag that every
- bound instance carries.
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass
- >>>> c = ConcreteClass()
- >>>> c._python_owns # True: object created in Python
- True
- >>>>
-
-* **methods**: Are represented as python methods and work as expected.
- They are first class objects and can be bound to an instance.
- Virtual C++ methods work as expected.
- To select a specific virtual method, do like with normal python classes
- that override methods: select it from the class that you need, rather than
- calling the method on the instance.
- To select a specific overload, use the __dispatch__ special function, which
- takes the name of the desired method and its signature (which can be
- obtained from the doc string) as arguments.
-
-* **namespaces**: Are represented as python classes.
- Namespaces are more open-ended than classes, so sometimes initial access may
- result in updates as data and functions are looked up and constructed
- lazily.
- Thus the result of ``dir()`` on a namespace shows the classes available,
- even if they may not have been created yet.
- It does not show classes that could potentially be loaded by the class
- loader.
- Once created, namespaces are registered as modules, to allow importing from
- them.
- Namespace currently do not work with the class loader.
- Fixing these bootstrap problems is on the TODO list.
- The global namespace is ``cppyy.gbl``.
-
-* **NULL**: Is represented as ``cppyy.gbl.nullptr``.
- In C++11, the keyword ``nullptr`` is used to represent ``NULL``.
- For clarity of intent, it is recommended to use this instead of ``None``
- (or the integer ``0``, which can serve in some cases), as ``None`` is better
- understood as ``void`` in C++.
-
-* **operator conversions**: If defined in the C++ class and a python
- equivalent exists (i.e. all builtin integer and floating point types, as well
- as ``bool``), it will map onto that python conversion.
- Note that ``char*`` is mapped onto ``__str__``.
- Example::
-
- >>>> from cppyy.gbl import ConcreteClass
- >>>> print ConcreteClass()
- Hello operator const char*!
- >>>>
-
-* **operator overloads**: If defined in the C++ class and if a python
- equivalent is available (not always the case, think e.g. of ``operator||``),
- then they work as expected.
- Special care needs to be taken for global operator overloads in C++: first,
- make sure that they are actually reflected, especially for the global
- overloads for ``operator==`` and ``operator!=`` of STL vector iterators in
- the case of gcc (note that they are not needed to iterate over a vector).
- Second, make sure that reflection info is loaded in the proper order.
- I.e. that these global overloads are available before use.
-
-* **pointers**: For builtin data types, see arrays.
- For objects, a pointer to an object and an object looks the same, unless
- the pointer is a data member.
- In that case, assigning to the data member will cause a copy of the pointer
- and care should be taken about the object's life time.
- If a pointer is a global variable, the C++ side can replace the underlying
- object and the python side will immediately reflect that.
-
-* **PyObject***: Arguments and return types of ``PyObject*`` can be used, and
- passed on to CPython API calls.
- Since these CPython-like objects need to be created and tracked (this all
- happens through ``cpyext``) this interface is not particularly fast.
-
-* **static data members**: Are represented as python property objects on the
- class and the meta-class.
- Both read and write access is as expected.
-
-* **static methods**: Are represented as python's ``staticmethod`` objects
- and can be called both from the class as well as from instances.
-
-* **strings**: The std::string class is considered a builtin C++ type and
- mixes quite well with python's str.
- Python's str can be passed where a ``const char*`` is expected, and an str
- will be returned if the return type is ``const char*``.
-
-* **templated classes**: Are represented in a meta-class style in python.
- This may look a little bit confusing, but conceptually is rather natural.
- For example, given the class ``std::vector<int>``, the meta-class part would
- be ``std.vector``.
- Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
- create an instance of that class, do ``std.vector(int)()``::
-
- >>>> import cppyy
- >>>> cppyy.load_reflection_info('libexampleDict.so')
- >>>> cppyy.gbl.std.vector # template metatype
- <cppyy.CppyyTemplateType object at 0x00007fcdd330f1a0>
- >>>> cppyy.gbl.std.vector(int) # instantiates template -> class
- <class '__main__.std::vector<int>'>
- >>>> cppyy.gbl.std.vector(int)() # instantiates class -> object
- <__main__.std::vector<int> object at 0x00007fe480ba4bc0>
- >>>>
-
- Note that templates can be build up by handing actual types to the class
- instantiation (as done in this vector example), or by passing in the list of
- template arguments as a string.
- The former is a lot easier to work with if you have template instantiations
- using classes that themselves are templates in the arguments (think e.g a
- vector of vectors).
- All template classes must already exist in the loaded reflection info, they
- do not work (yet) with the class loader.
-
- For compatibility with other bindings generators, use of square brackets
- instead of parenthesis to instantiate templates is supported as well.
-
-* **templated functions**: Automatically participate in overloading and are
- used in the same way as other global functions.
-
-* **templated methods**: For now, require an explicit selection of the
- template parameters.
- This will be changed to allow them to participate in overloads as expected.
-
-* **typedefs**: Are simple python references to the actual classes to which
- they refer.
-
-* **unary operators**: Are supported if a python equivalent exists, and if the
- operator is defined in the C++ class.
-
-You can always find more detailed examples and see the full of supported
-features by looking at the tests in pypy/module/cppyy/test.
-
-If a feature or reflection info is missing, this is supposed to be handled
-gracefully.
-In fact, there are unit tests explicitly for this purpose (even as their use
-becomes less interesting over time, as the number of missing features
-decreases).
-Only when a missing feature is used, should there be an exception.
-For example, if no reflection info is available for a return type, then a
-class that has a method with that return type can still be used.
-Only that one specific method can not be used.
-
-
-Templates
----------
-
-Templates can be automatically instantiated, assuming the appropriate header
-files have been loaded or are accessible to the class loader.
-This is the case for example for all of STL.
-For example::
-
- $ cat MyTemplate.h
- #include <vector>
-
- class MyClass {
- public:
- MyClass(int i = -99) : m_i(i) {}
- MyClass(const MyClass& s) : m_i(s.m_i) {}
- MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
- ~MyClass() {}
- int m_i;
- };
-
-Run the normal ``genreflex`` and compilation steps::
-
- $ genreflex MyTemplate.h --selection=MyTemplate.xml
- $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include
MyTemplate_rflx.cpp -o libTemplateDict.so -L$CPPYYHOME/lib -lCling
-
-Subsequent use should be as expected.
-Note the meta-class style of "instantiating" the template::
-
- >>>> import cppyy
- >>>> cppyy.load_reflection_info("libTemplateDict.so")
- >>>> std = cppyy.gbl.std
- >>>> MyClass = cppyy.gbl.MyClass
- >>>> v = std.vector(MyClass)()
- >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
- >>>> for m in v:
- .... print m.m_i,
- ....
- 1 2 3
- >>>>
-
-The arguments to the template instantiation can either be a string with the
-full list of arguments, or the explicit classes.
-The latter makes for easier code writing if the classes passed to the
-instantiation are themselves templates.
-
-
-The fast lane
--------------
-
-By default, cppyy will use direct function pointers through `CFFI`_ whenever
-possible. If this causes problems for you, you can disable it by setting the
-CPPYY_DISABLE_FASTPATH environment variable.
-
-.. _CFFI: https://cffi.readthedocs.io/en/latest/
-
-
-CPython
--------
-
-Most of the ideas in cppyy come originally from the `PyROOT`_ project, which
-contains a CPython-based cppyy.py module (with similar dependencies as the
-one that comes with PyPy).
-A standalone pip-installable version is planned, but for now you can install
-ROOT through your favorite distribution installer (available in the science
-section).
-
-.. _PyROOT: https://root.cern.ch/pyroot
-
-There are a couple of minor differences between the two versions of cppyy
-(the CPython version has a few more features).
-Work is on-going to integrate the nightly tests of both to make sure their
-feature sets are equalized.
-
-
-Python3
--------
-
-The CPython version of cppyy supports Python3, assuming your packager has
-build the backend for it.
-The cppyy module has not been tested with the `Py3k`_ version of PyPy.
-Note that the generated reflection information (from ``genreflex``) is fully
-independent of Python, and does not need to be rebuild when switching versions
-or interpreters.
-
-.. _Py3k: https://bitbucket.org/pypy/pypy/src/py3k
-
-
-.. toctree::
- :hidden:
-
- cppyy_example
diff --git a/pypy/doc/cppyy_example.rst b/pypy/doc/cppyy_example.rst
deleted file mode 100644
--- a/pypy/doc/cppyy_example.rst
+++ /dev/null
@@ -1,59 +0,0 @@
-File example.h
-==============
-
-::
-
- #include <iostream>
- #include <vector>
-
- class AbstractClass {
- public:
- virtual ~AbstractClass() {}
- virtual void abstract_method() = 0;
- };
-
- class ConcreteClass : AbstractClass {
- public:
- ConcreteClass(int n=42) : m_int(n) {}
- ~ConcreteClass() {}
-
- virtual void abstract_method() {
- std::cout << "called concrete method" << std::endl;
- }
-
- void array_method(int* ad, int size) {
- for (int i=0; i < size; ++i)
- std::cout << ad[i] << ' ';
- std::cout << std::endl;
- }
-
- void array_method(double* ad, int size) {
- for (int i=0; i < size; ++i)
- std::cout << ad[i] << ' ';
- std::cout << std::endl;
- }
-
- AbstractClass* show_autocast() {
- return this;
- }
-
- operator const char*() {
- return "Hello operator const char*!";
- }
-
- public:
- int m_int;
- };
-
- namespace Namespace {
-
- class ConcreteClass {
- public:
- class NestedClass {
- public:
- std::vector<int> m_v;
- };
-
- };
-
- } // namespace Namespace
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -337,6 +337,8 @@
- ``frozenset`` (empty frozenset only)
+ - unbound method objects (for Python 2 only)
+
This change requires some changes to ``id`` as well. ``id`` fulfills the
following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
above types will return a value that is computed from the argument, and can
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -12,7 +12,7 @@
* Write them in pure Python and use ctypes_.
-* Write them in C++ and bind them through :doc:`cppyy <cppyy>` using Cling.
+* Write them in C++ and bind them through cppyy_ using Cling.
* Write them as `RPython mixed modules`_.
@@ -61,29 +61,22 @@
.. _libffi: http://sourceware.org/libffi/
-Cling and cppyy
----------------
+cppyy
+-----
-The builtin :doc:`cppyy <cppyy>` module uses reflection information, provided
by
-`Cling`_ (which needs to be `installed separately`_), of C/C++ code to
-automatically generate bindings at runtime.
-In Python, classes and functions are always runtime structures, so when they
-are generated matters not for performance.
-However, if the backend itself is capable of dynamic behavior, it is a much
-better functional match, allowing tighter integration and more natural
-language mappings.
+For C++, _cppyy_ is an automated bindings generator available for both
+PyPy and CPython.
+_cppyy_ relies on declarations from C++ header files to dynamically
+construct Python equivalent classes, functions, variables, etc.
+It is designed for use by large scale programs and supports modern C++.
+With PyPy, it leverages the built-in ``_cppyy`` module, allowing the JIT to
+remove most of the cross-language overhead.
-The :doc:`cppyy <cppyy>` module is written in RPython, thus PyPy's JIT is able
to remove
-most cross-language call overhead.
+To install, run ``pip install cppyy``.
+Further details are available in the `full documentation`_.
-:doc:Full details are `available here <cppyy>`.
+.. _`full documentation`: https://cppyy.readthedocs.org/
-.. _installed separately: https://pypi.python.org/pypi/PyPy-cppyy-backend
-.. _Cling: https://root.cern.ch/cling
-
-.. toctree::
-
- cppyy
RPython Mixed Modules
---------------------
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -35,8 +35,8 @@
* Edit things. Use ``hg diff`` to see what you changed. Use ``hg add``
to make Mercurial aware of new files you added, e.g. new test files.
- Use ``hg status`` to see if there are such files. Run tests! (See
- the rest of this page.)
+ Use ``hg status`` to see if there are such files. Write and run tests!
+ (See the rest of this page.)
* Commit regularly with ``hg commit``. A one-line commit message is
fine. We love to have tons of commits; make one as soon as you have
@@ -113,6 +113,10 @@
make sure you have the correct version installed which
you can find out with the ``--version`` switch.
+You will need the `build requirements`_ to run tests successfully, since many
of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter
+
Now on to running some tests. PyPy has many different test directories
and you can use shell completion to point at directories or files::
@@ -141,7 +145,7 @@
.. _py.test testing tool: http://pytest.org
.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-
+.. _`build requirements`: build.html#install-build-time-dependencies
Special Introspection Features of the Untranslated Python Interpreter
---------------------------------------------------------------------
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -40,6 +40,9 @@
sure things are ported back to the trunk and to the branch as
necessary.
+* Maybe bump the SOABI number in module/imp/importing. This has many
+ implications, so make sure the PyPy community agrees to the change.
+
* Update and write documentation
* update pypy/doc/contributor.rst (and possibly LICENSE)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,14 @@
.. this is a revision shortly after release-pypy2.7-v5.8.0
.. startrev: 558bd00b3dd8
+In previous versions of PyPy, ``instance.method`` would return always
+the same bound method object, when gotten out of the same instance (as
+far as ``is`` and ``id()`` can tell). CPython doesn't do that. Now
+PyPy, like CPython, returns a different bound method object every time.
+For ``type.method``, PyPy2 still returns always the same *unbound*
+method object; CPython does it for built-in types but not for
+user-defined types.
+
.. branch: cffi-complex
.. branch: cffi-char16-char32
@@ -25,3 +33,43 @@
.. branch: cpyext-hash_notimpl
If ``tp_hash`` is ``PyObject_HashNotImplemented``, set
``obj.__dict__['__hash__']`` to None
+
+.. branch: cppyy-packaging
+
+Renaming of ``cppyy`` to ``_cppyy``.
+The former is now an external package installable with ``pip install cppyy``.
+
+.. branch: Enable_PGO_for_clang
+
+.. branch: nopax
+
+At the end of translation, run ``attr -q -s pax.flags -V m`` on
+PAX-enabled systems on the produced binary. This seems necessary
+because PyPy uses a JIT.
+
+.. branch: pypy_bytearray
+
+Improve ``bytearray`` performance (backported from py3.5)
+
+.. branch: gc-del-limit-growth
+
+Fix the bounds in the GC when allocating a lot of objects with finalizers,
+fixes issue #2590
+
+.. branch: arrays-force-less
+
+Small improvement to optimize list accesses with constant indexes better by
+throwing away information about them less eagerly.
+
+
+.. branch: getarrayitem-into-bridges:
+
+More information is retained into a bridge: knowledge about the content of
+arrays (at fixed indices) is stored in guards (and thus available at the
+beginning of bridges). Also, some better feeding of information about known
+fields of constant objects into bridges.
+
+.. branch: cpyext-leakchecking
+
+Add support for leakfinder in cpyext tests (disabled for now, due to too many
+failures).
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -2,6 +2,7 @@
Arguments objects.
"""
from rpython.rlib.debug import make_sure_not_resized
+from rpython.rlib.objectmodel import not_rpython
from rpython.rlib import jit
from rpython.rlib.objectmodel import enforceargs
from rpython.rlib.rstring import StringBuilder
@@ -48,8 +49,8 @@
# behaviour but produces better error messages
self.methodcall = methodcall
+ @not_rpython
def __repr__(self):
- """ NOT_RPYTHON """
name = self.__class__.__name__
if not self.keywords:
return '%s(%s)' % (name, self.arguments_w,)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1,4 +1,5 @@
import sys
+import py
from rpython.rlib.cache import Cache
from rpython.tool.uid import HUGEVAL_BYTES
@@ -1271,8 +1272,22 @@
self.setitem(w_globals, w_key, self.builtin)
return statement.exec_code(self, w_globals, w_locals)
+ @not_rpython
+ def appdef(self, source):
+ '''Create interp-level function object from app-level source.
+
+ The source should be in the same format as for space.appexec():
+ """(foo, bar): return 'baz'"""
+ '''
+ source = source.lstrip()
+ assert source.startswith('('), "incorrect header in:\n%s" % (source,)
+ source = py.code.Source("def anonymous%s\n" % source)
+ w_glob = self.newdict(module=True)
+ self.exec_(str(source), w_glob, w_glob)
+ return self.getitem(w_glob, self.newtext('anonymous'))
+
@specialize.arg(2)
- def appexec(self, posargs_w, source):
+ def appexec(self, posargs_w, source, cache=True):
""" return value from executing given source at applevel.
The source must look like
'''(x, y):
@@ -1280,7 +1295,11 @@
return result
'''
"""
- w_func = self.fromcache(AppExecCache).getorbuild(source)
+ if cache:
+ w_func = self.fromcache(AppExecCache).getorbuild(source)
+ else:
+ # NB: since appdef() is not-RPython, using cache=False also is.
+ w_func = self.appdef(source)
args = Arguments(self, list(posargs_w))
return self.call_args(w_func, args)
@@ -1817,15 +1836,7 @@
class AppExecCache(SpaceCache):
@not_rpython
def build(cache, source):
- space = cache.space
- # XXX will change once we have our own compiler
- import py
- source = source.lstrip()
- assert source.startswith('('), "incorrect header in:\n%s" % (source,)
- source = py.code.Source("def anonymous%s\n" % source)
- w_glob = space.newdict(module=True)
- space.exec_(str(source), w_glob, w_glob)
- return space.getitem(w_glob, space.newtext('anonymous'))
+ return cache.space.appdef(source)
# Table describing the regular part of the interface of object spaces,
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,7 +7,7 @@
from rpython.rlib import jit
from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.objectmodel import dont_inline, not_rpython
from rpython.rlib import rstack, rstackovf
from rpython.rlib import rwin32
from rpython.rlib import runicode
@@ -65,8 +65,9 @@
self.match(space, space.w_KeyboardInterrupt))
# note: an extra case is added in OpErrFmtNoArgs
+ @not_rpython
def __str__(self):
- "NOT_RPYTHON: Convenience for tracebacks."
+ "Convenience for tracebacks."
s = self._w_value
space = getattr(self.w_type, 'space', None)
if space is not None:
@@ -119,15 +120,16 @@
if RECORD_INTERPLEVEL_TRACEBACK:
self.debug_excs.append(sys.exc_info())
+ @not_rpython
def print_application_traceback(self, space, file=None):
- "NOT_RPYTHON: Dump a standard application-level traceback."
+ "Dump a standard application-level traceback."
if file is None:
file = sys.stderr
self.print_app_tb_only(file)
print >> file, self.errorstr(space)
+ @not_rpython
def print_app_tb_only(self, file):
- "NOT_RPYTHON"
tb = self._application_traceback
if tb:
import linecache
@@ -154,8 +156,9 @@
print >> file, l
tb = tb.next
+ @not_rpython
def print_detailed_traceback(self, space=None, file=None):
- """NOT_RPYTHON: Dump a nice detailed interpreter- and
+ """Dump a nice detailed interpreter- and
application-level traceback, useful to debug the interpreter."""
if file is None:
file = sys.stderr
diff --git a/pypy/interpreter/executioncontext.py
b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,6 +1,7 @@
import sys
from pypy.interpreter.error import OperationError, get_cleared_operation_error
from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize, not_rpython
from rpython.rlib import jit, rgc, objectmodel
TICK_COUNTER_STEP = 100
@@ -410,8 +411,9 @@
# to run at the next possible bytecode
self.reset_ticker(-1)
+ @not_rpython
def register_periodic_action(self, action, use_bytecode_counter):
- """NOT_RPYTHON:
+ """
Register the PeriodicAsyncAction action to be called whenever the
tick counter becomes smaller than 0. If 'use_bytecode_counter' is
True, make sure that we decrease the tick counter at every bytecode.
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -516,8 +516,9 @@
def __init__(self, space, w_function, w_instance):
self.space = space
+ assert w_instance is not None # unbound methods only exist in Python
2
self.w_function = w_function
- self.w_instance = w_instance # or None
+ self.w_instance = w_instance
def descr_method__new__(space, w_subtype, w_function, w_instance):
if space.is_w(w_instance, space.w_None):
@@ -577,24 +578,6 @@
return space.w_False
return space.newbool(space.eq_w(self.w_function, w_other.w_function))
- def is_w(self, space, other):
- if not isinstance(other, Method):
- return False
- return (self.w_instance is other.w_instance and
- self.w_function is other.w_function)
-
- def immutable_unique_id(self, space):
- from pypy.objspace.std.util import IDTAG_METHOD as tag
- from pypy.objspace.std.util import IDTAG_SHIFT
- if self.w_instance is not None:
- id = space.bigint_w(space.id(self.w_instance))
- id = id.lshift(LONG_BIT)
- else:
- id = rbigint.fromint(0)
- id = id.or_(space.bigint_w(space.id(self.w_function)))
- id = id.lshift(IDTAG_SHIFT).int_or_(tag)
- return space.newlong_from_rbigint(id)
-
def descr_method_hash(self):
space = self.space
w_result = space.hash(self.w_function)
@@ -606,7 +589,7 @@
from pypy.interpreter.gateway import BuiltinCode
w_mod = space.getbuiltinmodule('_pickle_support')
mod = space.interp_w(MixedModule, w_mod)
- w_instance = self.w_instance or space.w_None
+ w_instance = self.w_instance
w_function = self.w_function
if (isinstance(w_function, Function) and
isinstance(w_function.code, BuiltinCode)):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -23,7 +23,7 @@
DescrMismatch)
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.function import ClassMethod, FunctionWithFixedCode
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, not_rpython
from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
from rpython.tool.sourcetools import func_with_new_name, compile2
@@ -75,8 +75,8 @@
def _freeze_(self):
return True
+ @not_rpython
def unwrap(self, space, w_value):
- """NOT_RPYTHON"""
raise NotImplementedError
@@ -399,8 +399,8 @@
class BuiltinActivation(object):
_immutable_ = True
+ @not_rpython
def __init__(self, behavior):
- """NOT_RPYTHON"""
self.behavior = behavior
def _run(self, space, scope_w):
@@ -654,9 +654,9 @@
# When a BuiltinCode is stored in a Function object,
# you get the functionality of CPython's built-in function type.
+ @not_rpython
def __init__(self, func, unwrap_spec=None, self_type=None,
descrmismatch=None, doc=None):
- "NOT_RPYTHON"
# 'implfunc' is the interpreter-level function.
# Note that this uses a lot of (construction-time) introspection.
Code.__init__(self, func.__name__)
@@ -1004,10 +1004,10 @@
instancecache = {}
+ @not_rpython
def __new__(cls, f, app_name=None, unwrap_spec=None, descrmismatch=None,
as_classmethod=False, doc=None):
- "NOT_RPYTHON"
# f must be a function whose name does NOT start with 'app_'
self_type = None
if hasattr(f, 'im_func'):
@@ -1047,8 +1047,8 @@
return self
+ @not_rpython
def _getdefaults(self, space):
- "NOT_RPYTHON"
alldefs_w = {}
assert len(self._code._argnames) == len(self._code._unwrap_spec)
for name, spec in zip(self._code._argnames, self._code._unwrap_spec):
@@ -1124,8 +1124,8 @@
class GatewayCache(SpaceCache):
+ @not_rpython
def build(cache, gateway):
- "NOT_RPYTHON"
space = cache.space
defs_w, kw_defs_w = gateway._getdefaults(space)
code = gateway._code
@@ -1196,8 +1196,8 @@
w_globals = self.getwdict(space)
return space.getitem(w_globals, space.newtext(name))
+ @not_rpython
def interphook(self, name):
- "NOT_RPYTHON"
def appcaller(space, *args_w):
if not isinstance(space, ObjSpace):
raise TypeError("first argument must be a space instance.")
@@ -1234,15 +1234,16 @@
"""NOT_RPYTHON
The cache mapping each applevel instance to its lazily built w_dict"""
+ @not_rpython
def build(self, app):
- "NOT_RPYTHON. Called indirectly by Applevel.getwdict()."
+ "Called indirectly by Applevel.getwdict()."
return build_applevel_dict(app, self.space)
# __________ pure applevel version __________
+@not_rpython
def build_applevel_dict(self, space):
- "NOT_RPYTHON"
w_glob = space.newdict(module=True)
space.setitem(w_glob, space.newtext('__name__'),
space.newtext(self.modname))
space.exec_(self.source, w_glob, w_glob,
@@ -1253,8 +1254,9 @@
# ____________________________________________________________
+@not_rpython
def appdef(source, applevel=ApplevelClass, filename=None):
- """ NOT_RPYTHON: build an app-level helper function, like for example:
+ """ build an app-level helper function, like for example:
myfunc = appdef('''myfunc(x, y):
return x+y
''')
@@ -1300,6 +1302,6 @@
# app2interp_temp is used for testing mainly
+@not_rpython
def app2interp_temp(func, applevel_temp=applevel_temp, filename=None):
- """ NOT_RPYTHON """
return appdef(func, applevel_temp, filename=filename)
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -3,6 +3,7 @@
"""
from rpython.rlib.listsort import make_timsort_class
+from rpython.rlib.objectmodel import not_rpython
class ThreadLocals:
@@ -41,9 +42,8 @@
# but in some corner cases it is not... unsure why
self._value = None
-
+@not_rpython
def make_weak_value_dictionary(space, keytype, valuetype):
- "NOT_RPYTHON"
if space.config.translation.rweakref:
from rpython.rlib.rweakref import RWeakValueDictionary
return RWeakValueDictionary(keytype, valuetype)
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -3,6 +3,9 @@
from pypy.interpreter import gateway
from pypy.interpreter.error import OperationError
from pypy.interpreter.baseobjspace import W_Root
+
+from rpython.rlib.objectmodel import not_rpython
+
import sys
class MixedModule(Module):
@@ -15,8 +18,8 @@
lazy = False
submodule_name = None
+ @not_rpython
def __init__(self, space, w_name):
- """ NOT_RPYTHON """
Module.__init__(self, space, w_name)
init_extra_module_attrs(space, self)
self.lazy = True
@@ -25,8 +28,9 @@
self.loaders = self.loaders.copy() # copy from the class to the inst
self.submodules_w = []
+ @not_rpython
def install(self):
- """NOT_RPYTHON: install this module, and it's submodules into
+ """install this module, and it's submodules into
space.builtin_modules"""
Module.install(self)
if hasattr(self, "submodules"):
@@ -66,8 +70,8 @@
self.w_initialdict = self.space.call_method(w_dict, 'copy')
@classmethod
+ @not_rpython
def get_applevel_name(cls):
- """ NOT_RPYTHON """
if cls.applevel_name is not None:
return cls.applevel_name
else:
@@ -163,8 +167,8 @@
self._frozen = True
@classmethod
+ @not_rpython
def buildloaders(cls):
- """ NOT_RPYTHON """
if not hasattr(cls, 'loaders'):
# build a constant dictionary out of
# applevel/interplevel definitions
@@ -194,8 +198,8 @@
return space.newtext_or_none(cls.__doc__)
+@not_rpython
def getinterpevalloader(pkgroot, spec):
- """ NOT_RPYTHON """
def ifileloader(space):
d = {'space': space}
# EVIL HACK (but it works, and this is not RPython :-)
@@ -235,8 +239,8 @@
return ifileloader
applevelcache = {}
+@not_rpython
def getappfileloader(pkgroot, appname, spec):
- """ NOT_RPYTHON """
# hum, it's a bit more involved, because we usually
# want the import at applevel
modname, attrname = spec.split('.')
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -4,7 +4,7 @@
from pypy.interpreter.baseobjspace import W_Root
from pypy.interpreter.error import OperationError, oefmt
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, not_rpython
class Module(W_Root):
@@ -35,8 +35,9 @@
except OperationError:
pass
+ @not_rpython
def install(self):
- """NOT_RPYTHON: installs this module into space.builtin_modules"""
+ """installs this module into space.builtin_modules"""
modulename = self.space.text0_w(self.w_name)
if modulename in self.space.builtin_modules:
raise ValueError(
@@ -44,8 +45,9 @@
"app-level module %r" % (modulename,))
self.space.builtin_modules[modulename] = self
+ @not_rpython
def setup_after_space_initialization(self):
- """NOT_RPYTHON: to allow built-in modules to do some more setup
+ """to allow built-in modules to do some more setup
after the space is fully initialized."""
def init(self, space):
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -7,6 +7,7 @@
from rpython.rlib.debug import ll_assert_not_none
from rpython.rlib.jit import hint
from rpython.rlib.objectmodel import instantiate, specialize, we_are_translated
+from rpython.rlib.objectmodel import not_rpython
from rpython.rlib.rarithmetic import intmask, r_uint
from rpython.tool.pairtype import extendabletype
@@ -146,8 +147,9 @@
return None
return d.w_locals
+ @not_rpython
def __repr__(self):
- # NOT_RPYTHON: useful in tracebacks
+ # useful in tracebacks
return "<%s.%s executing %s at line %s" % (
self.__class__.__module__, self.__class__.__name__,
self.pycode, self.get_last_lineno())
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -7,7 +7,7 @@
from rpython.rlib import jit, rstackovf, rstring
from rpython.rlib.debug import check_nonneg
from rpython.rlib.objectmodel import (
- we_are_translated, always_inline, dont_inline)
+ we_are_translated, always_inline, dont_inline, not_rpython)
from rpython.rlib.rarithmetic import r_uint, intmask
from rpython.tool.sourcetools import func_with_new_name
@@ -23,8 +23,8 @@
CANNOT_CATCH_MSG = ("catching classes that don't inherit from BaseException "
"is not allowed in 3.x")
+@not_rpython
def unaryoperation(operationname):
- """NOT_RPYTHON"""
def opimpl(self, *ignored):
operation = getattr(self.space, operationname)
w_1 = self.popvalue()
@@ -34,8 +34,8 @@
return func_with_new_name(opimpl, "opcode_impl_for_%s" % operationname)
+@not_rpython
def binaryoperation(operationname):
- """NOT_RPYTHON"""
def opimpl(self, *ignored):
operation = getattr(self.space, operationname)
w_2 = self.popvalue()
diff --git a/pypy/interpreter/test/test_function.py
b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -1,5 +1,5 @@
# encoding: utf-8
-import pytest
+import pytest, sys
from pypy.interpreter import eval
from pypy.interpreter.function import Function, Method, descr_function_get
from pypy.interpreter.pycode import PyCode
@@ -416,6 +416,11 @@
raises(ValueError, FunctionType.__setstate__, f, (1, 2, 3))
class AppTestMethod:
+ def setup_class(cls):
+ cls.w_runappdirect_on_cpython = cls.space.wrap(
+ cls.runappdirect and
+ '__pypy__' not in sys.builtin_module_names)
+
def test_simple_call(self):
class A(object):
def func(self, arg2):
@@ -586,7 +591,6 @@
assert meth == meth
assert meth == MethodType(func, object)
- @pytest.mark.skipif("config.option.runappdirect")
def test_method_identity(self):
class A(object):
def m(self):
@@ -603,19 +607,24 @@
a = A()
a2 = A()
- assert a.m is a.m
- assert id(a.m) == id(a.m)
- assert a.m is not a.n
- assert id(a.m) != id(a.n)
- assert a.m is not a2.m
- assert id(a.m) != id(a2.m)
+ x = a.m; y = a.m
+ assert x is not y
+ assert id(x) != id(y)
+ assert x == y
+ assert x is not a.n
+ assert id(x) != id(a.n)
+ assert x is not a2.m
+ assert id(x) != id(a2.m)
- assert A.m is A.m
- assert id(A.m) == id(A.m)
- assert A.m is not A.n
- assert id(A.m) != id(A.n)
- assert A.m is B.m
- assert id(A.m) == id(B.m)
+ if not self.runappdirect_on_cpython:
+ assert A.m is A.m
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
