Author: Armin Rigo <[email protected]>
Branch: reverse-debugger
Changeset: r94653:dd287c100ac7
Date: 2018-05-23 16:12 +0200
http://bitbucket.org/pypy/pypy/changeset/dd287c100ac7/
Log: hg merge default
diff too long, truncating to 2000 out of 4168 lines
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -56,3 +56,5 @@
0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
+ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
+fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt
rename from dotviewer/font/NOTICE
rename to dotviewer/font/COPYING.txt
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -82,8 +82,11 @@
def _CData_output(self, resarray, base=None, index=-1):
from _rawffi.alt import types
# If a char_p or unichar_p is received, skip the string interpretation
- if base._ffiargtype != types.Pointer(types.char_p) and \
- base._ffiargtype != types.Pointer(types.unichar_p):
+ try:
+ deref = type(base)._deref_ffiargtype()
+ except AttributeError:
+ deref = None
+ if deref != types.char_p and deref != types.unichar_p:
# this seems to be a string if we're array of char, surprise!
from ctypes import c_char, c_wchar
if self._type_ is c_char:
@@ -120,6 +123,12 @@
value = self(*value)
return _CDataMeta.from_param(self, value)
+ def _build_ffiargtype(self):
+ return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+ def _deref_ffiargtype(self):
+ return self._type_.get_ffi_argtype()
+
def array_get_slice_params(self, index):
if hasattr(self, '_length_'):
start, stop, step = index.indices(self._length_)
@@ -248,6 +257,5 @@
_type_ = base
)
cls = ArrayMeta(name, (Array,), tpdict)
- cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype())
ARRAY_CACHE[key] = cls
return cls
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -49,10 +49,13 @@
else:
return self.from_param(as_parameter)
+ def _build_ffiargtype(self):
+ return _shape_to_ffi_type(self._ffiargshape_)
+
def get_ffi_argtype(self):
if self._ffiargtype:
return self._ffiargtype
- self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_)
+ self._ffiargtype = self._build_ffiargtype()
return self._ffiargtype
def _CData_output(self, resbuffer, base=None, index=-1):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -70,7 +70,12 @@
self._ffiarray = ffiarray
self.__init__ = __init__
self._type_ = TP
- self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
+
+ def _build_ffiargtype(self):
+ return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+ def _deref_ffiargtype(self):
+ return self._type_.get_ffi_argtype()
from_address = cdata_from_address
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -160,6 +160,10 @@
raise AttributeError("_fields_ is final")
if self in [f[1] for f in value]:
raise AttributeError("Structure or union cannot contain itself")
+ if self._ffiargtype is not None:
+ raise NotImplementedError("Too late to set _fields_: we already "
+ "said to libffi that the structure type %s is opaque"
+ % (self,))
names_and_fields(
self,
value, self.__bases__[0],
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
This division between bytecode evaluator and object space gives a lot of
flexibility. One can plug in different :doc:`object spaces <objspace>` to get
different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write
interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory. ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython. The fact that it is
+RPython is not apparent at first. Built-in modules are written in
+``pypy/module/*``. Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory. The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that
traces the
+interpreter written in RPython, rather than the user program that it
+interprets. As a result it applies to any interpreter, i.e. any
+language. But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter. The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code. Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation. Moreover, this is not reference counting; it is a real
+GC written as more RPython code. The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`:
http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
* PyPy 2.5.1 or earlier: normal users would see permission errors.
Installers need to run ``pypy -c "import gdbm"`` and other similar
commands at install time; the exact list is in
- :source:`pypy/tool/release/package.py <package.py>`. Users
+ :source:`pypy/tool/release/package.py`. Users
seeing a broken installation of PyPy can fix it after-the-fact if they
have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
* PyPy 2.6 and later: anyone would get ``ImportError: no module named
_gdbm_cffi``. Installers need to run ``pypy _gdbm_build.py`` in the
``lib_pypy`` directory during the installation process (plus others;
- see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+ see the exact list in :source:`pypy/tool/release/package.py`).
Users seeing a broken
installation of PyPy can fix it after-the-fact, by running ``pypy
/path/to/lib_pypy/_gdbm_build.py``. This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
hg help branch
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
.. _using-development-tracker:
@@ -547,15 +547,7 @@
Using the development bug/feature tracker
-----------------------------------------
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application. You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
.. _testing:
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
:maxdepth: 1
man/pypy.1.rst
+ man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
# built documents.
#
# The short X.Y version.
-version = '5.8'
+version = '6.0'
# The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt
b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt
b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt
b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension
modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,472 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are
three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+ separated from each other. More about this below, but often the manifestation
+ of this is that things are at a different layer than you would expect them
+ to be. For example if you are looking for the JIT implementation, you will
+ not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+ It's not only what we believe in, but also that PyPy's architecture is
+ working very well with TDD in mind and not so well without it. Often
+ development means progressing in an unrelated corner, one unittest
+ at a time; and then flipping a giant switch, bringing it all together.
+ (It generally works out of the box. If it doesn't, then we didn't
+ write enough unit tests.) It's worth repeating - PyPy's
+ approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+ in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a
runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC
channel,
+details of which can be found in our :ref:`contact <contact>` section. The
folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do
something
+with PyPy, you can become a committer. We also run frequent coding sprints
which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will
typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+
https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+ icons). You get a fork of the repository, e.g. in
+ `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+ ``hg clone ssh://[email protected]/yourname/pypy``. It is a very slow
+ operation but only ever needs to be done once. See also
+ http://pypy.org/download.html#building-from-source .
+ If you already cloned
+ ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+ then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+ your clone to contain the line ``default =
+ ssh://[email protected]/yourname/pypy``, and then do ``hg pull && hg
+ up``. If you already have such a clone but don't want to change it,
+ you can clone that copy with ``hg clone /path/to/other/copy``, and
+ then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo. Make a branch
+ with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things. Use ``hg diff`` to see what you changed. Use ``hg add``
+ to make Mercurial aware of new files you added, e.g. new test files.
+ Use ``hg status`` to see if there are such files. Write and run tests!
+ (See the rest of this page.)
+
+* Commit regularly with ``hg commit``. A one-line commit message is
+ fine. We love to have tons of commits; make one as soon as you have
+ some progress, even if it is only some new test that doesn't pass yet,
+ or fixing things even if not all tests pass. Step by step, you are
+ building the history of your changes, which is the point of a version
+ control system. (There are commands like ``hg log`` and ``hg up``
+ that you should read about later, to learn how to navigate this
+ history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+ them back to the repo named in the file ``.hg/hgrc``. Repos are
+ basically just collections of commits (a commit is also called a
+ changeset): there is one repo per url, plus one for each local copy on
+ each local machine. The commands ``hg push`` and ``hg pull`` copy
+ commits around, with the goal that all repos in question end up with
+ the exact same set of commits. By opposition, ``hg up`` only updates
+ the "working copy" by reading the local repository, i.e. it makes the
+ files that you see correspond to the latest (or any other) commit
+ locally present.
+
+* You should push often; there is no real reason not to. Remember that
+ even if they are pushed, with the setup above, the commits are (1)
+ only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+ named. Yes, they are publicly visible, but don't worry about someone
+ walking around the thousands of repos on bitbucket saying "hah, look
+ at the bad coding style of that guy". Try to get into the mindset
+ that your work is not secret and it's fine that way. We might not
+ accept it as is for PyPy, asking you instead to improve some things,
+ but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+ like to merge that branch back to the original ``pypy/pypy`` repo.
+ This can also be done several times if you have interesting
+ intermediate states, but if you get there, then we're likely to
+ proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+ ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+ Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+ ``default = ssh://[email protected]/pypy/pypy``. Your changes will
+ then be pushed directly to the official repo, but (if you follow these
+ rules) they are still on a branch, and we can still review the
+ branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+ that we generally push small changes as one or a few commits directly
+ to the branch ``default``. Also, we often collaborate even if we are
+ on other branches, which do not really "belong" to anyone. At this
+ point you'll need ``hg merge`` and learn how to resolve conflicts that
+ sometimes occur when two people try to push different commits in
+ parallel on the same branch. But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory). You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+* :source:`pypy/interpreter` contains the bytecode interpreter: bytecode
dispatcher
+ in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+ :source:`pypy/interpreter/eval.py` and
:source:`pypy/interpreter/pyframe.py`,
+ function objects and argument passing in
:source:`pypy/interpreter/function.py`
+ and :source:`pypy/interpreter/argument.py`, the object space interface
+ definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+ :source:`pypy/interpreter/module.py` and
:source:`pypy/interpreter/mixedmodule.py`.
+ Core types supporting the bytecode interpreter are defined in
+ :source:`pypy/interpreter/typedef.py`.
+
+* :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+ and grammar files that allow it to parse the syntax of various Python
+ versions. Once the grammar has been processed, the parser can be
+ translated by the above machinery into efficient code.
+
+* :source:`pypy/interpreter/astcompiler` contains the compiler. This
+ contains a modified version of the compiler package from CPython
+ that fixes some bugs and is translatable.
+
+* :source:`pypy/objspace/std` contains the
+ :ref:`Standard object space <standard-object-space>`. The main file
+ is :source:`pypy/objspace/std/objspace.py`. For each type, the file
+ ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+ as a first approximation. (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you
sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install
independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other
projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+ python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+ py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many
of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
+
+Now on to running some tests. PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+ py.test pypy/interpreter/test/test_pyframe.py
+
+ # or for running tests of a whole subdirectory
+ py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``. It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+ py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython. On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+ python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+ cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+ cd pypy
+ python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console. You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created
on the PyPy
+prompt with the prefix ``w_``::
+
+ >>>> a = 123
+ >>>> <Ctrl-C>
+ *** Entering interpreter-level console ***
+ >>> w_a
+ W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the
``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+ >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+ >>> <Ctrl-D>
+ *** Leaving interpreter-level console ***
+
+ KeyboardInterrupt
+ >>>> l
+ [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+ cd pypy
+ python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in
addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+ python pyinteractive.py --withmod-time -c "from test import pystone;
pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+ python pyinteractive.py --withmod-time
../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``. By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what
all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes. To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+ >>>> __pytrace__ = 1
+ Tracing enabled
+ >>>> x = 5
+ <module>: LOAD_CONST 0 (5)
+ <module>: STORE_NAME 0 (x)
+ <module>: LOAD_CONST 1 (None)
+ <module>: RETURN_VALUE 0
+ >>>> x
+ <module>: LOAD_NAME 0 (x)
+ <module>: PRINT_EXPR 0
+ 5
+ <module>: LOAD_CONST 0 (None)
+ <module>: RETURN_VALUE 0
+ >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+ graphviz: http://www.graphviz.org/Download.php
+
+ pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/discussion/ctypes-implementation.rst
b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
.. _pyglet: http://pyglet.org/
-
-ctypes configure
------------------
-
-We also released ``ctypes-configure``, which is an experimental package
-trying to approach the portability issues of ctypes-based code.
-
-idea
-~~~~
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes c
-compiler (via distutils) for various platform-dependent details like
-exact sizes of types (for example size_t), ``#defines``, exact outline of
-structures etc. It replaces in this regard code generator (h2py).
-
-installation
-~~~~~~~~~~~~
-
-``easy_install ctypes_configure``
-
-usage
-~~~~~
-
-:source:`ctypes_configure/doc/sample.py` explains in details how to use it.
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -1,5 +1,5 @@
-Embedding PyPy
-==============
+Embedding PyPy (DEPRECATED)
+===========================
PyPy has a very minimal and a very strange embedding interface, based on
the usage of `cffi`_ and the philosophy that Python is a better language than
diff --git a/pypy/doc/eventhistory.rst b/pypy/doc/eventhistory.rst
--- a/pypy/doc/eventhistory.rst
+++ b/pypy/doc/eventhistory.rst
@@ -40,11 +40,9 @@
Main focus of the sprint will be on the goals of the upcoming June 0.9
release.
-Read more in `the sprint announcement`__, see who is planning to attend
-on the `people page`_.
+Read more about `the sprint`__
-__
https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/announce.html
-.. _people page:
https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/people.txt
+__ https://bitbucket.org/pypy/extradoc/src/extradoc/sprintinfo/ddorf2006/
PyPy sprint at Akihabara (Tokyo, Japan)
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -75,12 +75,12 @@
.. _A Way Forward in Parallelising Dynamic Languages:
https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
.. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages:
https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
.. _Allocation Removal by Partial Evaluation in a Tracing JIT:
https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
-.. _Towards a Jitting VM for Prolog Execution:
http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
+.. _Towards a Jitting VM for Prolog Execution:
http://stups.hhu.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
.. _High performance implementation of Python for CLI/.NET with JIT compiler
generation for dynamic languages:
http://buildbot.pypy.org/misc/antocuni-thesis.pdf
.. _How to *not* write Virtual Machines for Dynamic Languages:
https://bitbucket.org/pypy/extradoc/raw/tip/talk/dyla2007/dyla.pdf
.. _`Tracing the Meta-Level: PyPy's Tracing JIT Compiler`:
https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009/bolz-tracing-jit.pdf
.. _`Faster than C#: Efficient Implementation of Dynamic Languages on .NET`:
https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009-dotnet/cli-jit.pdf
-.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation:
http://stups.hhu.de/mediawiki/images/b/b9/Master_bolz.pdf
+.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation:
https://www.researchgate.net/profile/Davide_Ancona/publication/252023163_Automatic_generation_of_JIT_compilers_for_dynamic_languages_in_NET/links/53f2098e0cf2bc0c40e70023/Automatic-generation-of-JIT-compilers-for-dynamic-languages-in-NET.pdf
.. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO
Languages`:
http://www.disi.unige.it/person/AnconaD/papers/DynamicLanguages_abstracts.html#AACM-DLS07
.. _EU Reports: index-report.html
.. _Hardware Transactional Memory Support for Lightweight Dynamic Language
Evolution: http://sabi.net/nriley/pubs/dls6-riley.pdf
@@ -368,6 +368,6 @@
.. _LLVM: http://llvm.org/
.. _IronPython: http://ironpython.codeplex.com/
.. _Dynamic Native Optimization of Native Interpreters:
http://people.csail.mit.edu/gregs/dynamorio.html
-.. _JikesRVM: http://jikesrvm.org/
+.. _JikesRVM: http://www.jikesrvm.org/
.. _Tunes: http://tunes.org
.. _old Tunes Wiki: http://buildbot.pypy.org/misc/cliki.tunes.org/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -67,7 +67,7 @@
you may need to run the command with `sudo` for a global installation.
The other commands of ``setup.py`` are available too, like ``build``.
-.. _PyPI: https://pypi.python.org/pypi
+.. _PyPI: https://pypi.org
.. _`use virtualenv (as documented here)`:
install.html#installing-using-virtualenv
@@ -360,7 +360,7 @@
(produced during a sprint). On the `PyPy bitbucket page`_ there is also a
Scheme and an Io implementation; both of these are unfinished at the moment.
-.. _Topaz: http://topazruby.com/
+.. _Topaz: http://docs.topazruby.com/en/latest/
.. _Hippy: http://morepypy.blogspot.ch/2012/07/hello-everyone.html
.. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/
.. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -152,7 +152,7 @@
to wait until it reaches a point in which the interpreter is in a known state
and calling user-defined code is harmless. It might happen that multiple
events occur before the hook is invoked: in this case, you can inspect the
-value ``stats.count`` to know how many times the event occured since the last
+value ``stats.count`` to know how many times the event occurred since the last
time the hook was called. Similarly, ``stats.duration`` contains the
**total** time spent by the GC for this specific event since the last time the
hook was called.
@@ -163,7 +163,7 @@
The attributes for ``GcMinorStats`` are:
``count``
- The number of minor collections occured since the last hook call.
+ The number of minor collections occurred since the last hook call.
``duration``
The total time spent inside minor collections since the last hook
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
deleted file mode 100644
--- a/pypy/doc/getting-started-dev.rst
+++ /dev/null
@@ -1,345 +0,0 @@
-Getting Started Developing With PyPy
-====================================
-
-.. contents::
-
-
-Using Mercurial
----------------
-
-PyPy development is based on Mercurial (hg). If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
-
-* Make an account on bitbucket_.
-
-* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
- icons). You get a fork of the repository, e.g. in
- https://bitbucket.org/yourname/pypy/.
-
-* Clone this new repo (i.e. the fork) to your local machine with the command
- ``hg clone ssh://[email protected]/yourname/pypy``. It is a very slow
- operation but only ever needs to be done once. See also
- http://pypy.org/download.html#building-from-source .
- If you already cloned
- ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
- then you can reuse the same clone by editing the file ``.hg/hgrc`` in
- your clone to contain the line ``default =
- ssh://[email protected]/yourname/pypy``, and then do ``hg pull && hg
- up``. If you already have such a clone but don't want to change it,
- you can clone that copy with ``hg clone /path/to/other/copy``, and
- then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
-
-* Now you have a complete copy of the PyPy repo. Make a branch
- with a command like ``hg branch name_of_your_branch``.
-
-* Edit things. Use ``hg diff`` to see what you changed. Use ``hg add``
- to make Mercurial aware of new files you added, e.g. new test files.
- Use ``hg status`` to see if there are such files. Write and run tests!
- (See the rest of this page.)
-
-* Commit regularly with ``hg commit``. A one-line commit message is
- fine. We love to have tons of commits; make one as soon as you have
- some progress, even if it is only some new test that doesn't pass yet,
- or fixing things even if not all tests pass. Step by step, you are
- building the history of your changes, which is the point of a version
- control system. (There are commands like ``hg log`` and ``hg up``
- that you should read about later, to learn how to navigate this
- history.)
-
-* The commits stay on your machine until you do ``hg push`` to "push"
- them back to the repo named in the file ``.hg/hgrc``. Repos are
- basically just collections of commits (a commit is also called a
- changeset): there is one repo per url, plus one for each local copy on
- each local machine. The commands ``hg push`` and ``hg pull`` copy
- commits around, with the goal that all repos in question end up with
- the exact same set of commits. By opposition, ``hg up`` only updates
- the "working copy" by reading the local repository, i.e. it makes the
- files that you see correspond to the latest (or any other) commit
- locally present.
-
-* You should push often; there is no real reason not to. Remember that
- even if they are pushed, with the setup above, the commits are (1)
- only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
- named. Yes, they are publicly visible, but don't worry about someone
- walking around the thousands of repos on bitbucket saying "hah, look
- at the bad coding style of that guy". Try to get into the mindset
- that your work is not secret and it's fine that way. We might not
- accept it as is for PyPy, asking you instead to improve some things,
- but we are not going to judge you.
-
-* The final step is to open a pull request, so that we know that you'd
- like to merge that branch back to the original ``pypy/pypy`` repo.
- This can also be done several times if you have interesting
- intermediate states, but if you get there, then we're likely to
- proceed to the next stage, which is...
-
-* Get a regular account for pushing directly to
- ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
- Once you have it you can rewrite your file ``.hg/hgrc`` to contain
- ``default = ssh://[email protected]/pypy/pypy``. Your changes will
- then be pushed directly to the official repo, but (if you follow these
- rules) they are still on a branch, and we can still review the
- branches you want to merge.
-
-* If you get closer to the regular day-to-day development, you'll notice
- that we generally push small changes as one or a few commits directly
- to the branch ``default``. Also, we often collaborate even if we are
- on other branches, which do not really "belong" to anyone. At this
- point you'll need ``hg merge`` and learn how to resolve conflicts that
- sometimes occur when two people try to push different commits in
- parallel on the same branch. But it is likely an issue for later ``:-)``
-
-.. _bitbucket: https://bitbucket.org/
-
-
-Running PyPy's unit tests
--------------------------
-
-PyPy development always was and is still thoroughly test-driven.
-We use the flexible `py.test testing tool`_ which you can `install
independently
-<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other
projects.
-
-The PyPy source tree comes with an inlined version of ``py.test``
-which you can invoke by typing::
-
- python pytest.py -h
-
-This is usually equivalent to using an installed version::
-
- py.test -h
-
-If you encounter problems with the installed version
-make sure you have the correct version installed which
-you can find out with the ``--version`` switch.
-
-You will need the `build requirements`_ to run tests successfully, since many
of
-them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
-
-Now on to running some tests. PyPy has many different test directories
-and you can use shell completion to point at directories or files::
-
- py.test pypy/interpreter/test/test_pyframe.py
-
- # or for running tests of a whole subdirectory
- py.test pypy/interpreter/
-
-See `py.test usage and invocations`_ for some more generic info
-on how you can run tests.
-
-Beware trying to run "all" pypy tests by pointing to the root
-directory or even the top level subdirectory ``pypy``. It takes
-hours and uses huge amounts of RAM and is not recommended.
-
-To run CPython regression tests you can point to the ``lib-python``
-directory::
-
- py.test lib-python/2.7/test/test_datetime.py
-
-This will usually take a long time because this will run
-the PyPy Python interpreter on top of CPython. On the plus
-side, it's usually still faster than doing a full translation
-and running the regression test with the translated PyPy Python
-interpreter.
-
-.. _py.test testing tool: http://pytest.org
-.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-.. _`build requirements`: build.html#install-build-time-dependencies
-
-Special Introspection Features of the Untranslated Python Interpreter
----------------------------------------------------------------------
-
-If you are interested in the inner workings of the PyPy Python interpreter,
-there are some features of the untranslated Python interpreter that allow you
-to introspect its internals.
-
-
-Interpreter-level console
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To start interpreting Python with PyPy, install a C compiler that is
-supported by distutils and use Python 2.7 or greater to run PyPy::
-
- cd pypy
- python bin/pyinteractive.py
-
-After a few seconds (remember: this is running on top of CPython), you should
-be at the PyPy prompt, which is the same as the Python prompt, but with an
-extra ">".
-
-If you press
-<Ctrl-C> on the console you enter the interpreter-level console, a
-usual CPython console. You can then access internal objects of PyPy
-(e.g. the :ref:`object space <objspace>`) and any variables you have created
on the PyPy
-prompt with the prefix ``w_``::
-
- >>>> a = 123
- >>>> <Ctrl-C>
- *** Entering interpreter-level console ***
- >>> w_a
- W_IntObject(123)
-
-The mechanism works in both directions. If you define a variable with the
``w_`` prefix on the interpreter-level, you will see it on the app-level::
-
- >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
- >>> <Ctrl-D>
- *** Leaving interpreter-level console ***
-
- KeyboardInterrupt
- >>>> l
- [1, 'abc']
-
-Note that the prompt of the interpreter-level console is only '>>>' since
-it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
-Linux) or <Ctrl-Z>, <Enter> (under Windows).
-
-Also note that not all modules are available by default in this mode (for
-example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
-``--withmod-...`` command line options.
-
-You may be interested in reading more about the distinction between
-:ref:`interpreter-level and app-level <interpreter-level>`.
-
-pyinteractive.py options
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To list the PyPy interpreter command line options, type::
-
- cd pypy
- python bin/pyinteractive.py --help
-
-pyinteractive.py supports most of the options that CPython supports too (in
addition to a
-large amount of options that can be used to customize pyinteractive.py).
-As an example of using PyPy from the command line, you could type::
-
- python pyinteractive.py --withmod-time -c "from test import pystone;
pystone.main(10)"
-
-Alternatively, as with regular Python, you can simply give a
-script name on the command line::
-
- python pyinteractive.py --withmod-time
../../lib-python/2.7/test/pystone.py 10
-
-The ``--withmod-xxx`` option enables the built-in module ``xxx``. By
-default almost none of them are, because initializing them takes time.
-If you want anyway to enable all built-in modules, you can use
-``--allworkingmodules``.
-
-See our :doc:`configuration sections <config/index>` for details about what
all the commandline
-options do.
-
-
-.. _trace example:
-
-Tracing bytecode and operations on objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use a simple tracing mode to monitor the interpretation of
-bytecodes. To enable it, set ``__pytrace__ = 1`` on the interactive
-PyPy console::
-
- >>>> __pytrace__ = 1
- Tracing enabled
- >>>> x = 5
- <module>: LOAD_CONST 0 (5)
- <module>: STORE_NAME 0 (x)
- <module>: LOAD_CONST 1 (None)
- <module>: RETURN_VALUE 0
- >>>> x
- <module>: LOAD_NAME 0 (x)
- <module>: PRINT_EXPR 0
- 5
- <module>: LOAD_CONST 0 (None)
- <module>: RETURN_VALUE 0
- >>>>
-
-
-Demos
------
-
-The `example-interpreter`_ repository contains an example interpreter
-written using the RPython translation toolchain.
-
-.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
-
-
-Additional Tools for running (and hacking) PyPy
------------------------------------------------
-
-We use some optional tools for developing PyPy. They are not required to run
-the basic tests or to get an interactive PyPy prompt but they help to
-understand and debug PyPy especially for the translation process.
-
-
-graphviz & pygame for flow graph viewing (highly recommended)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-graphviz and pygame are both necessary if you
-want to look at generated flow graphs:
-
- graphviz: http://www.graphviz.org/Download.php
-
- pygame: http://www.pygame.org/download.shtml
-
-
-py.test and the py lib
-~~~~~~~~~~~~~~~~~~~~~~
-
-The `py.test testing tool`_ drives all our testing needs.
-
-We use the `py library`_ for filesystem path manipulations, terminal
-writing, logging and some other support functionality.
-
-You don't necessarily need to install these two libraries because
-we also ship them inlined in the PyPy source tree.
-
-.. _py library: http://pylib.readthedocs.org/
-
-
-Getting involved
-----------------
-
-PyPy employs an open development process. You are invited to join our
-`pypy-dev mailing list`_ or look at the other :ref:`contact
-possibilities <contact>`. Usually we give out commit rights fairly liberally,
so if you
-want to do something with PyPy, you can become a committer. We also run
frequent
-coding sprints which are separately announced and often happen around Python
-conferences such as EuroPython or PyCon. Upcoming events are usually announced
-on `the blog`_.
-
-.. _the blog: http://morepypy.blogspot.com
-.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
-
-
-.. _start-reading-sources:
-
-Where to start reading the sources
-----------------------------------
-
-PyPy is made from parts that are relatively independent of each other.
-You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory). You may look at our
:doc:`directory reference <dir-reference>`
-or start off at one of the following points:
-
-* :source:`pypy/interpreter` contains the bytecode interpreter: bytecode
dispatcher
- in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
- :source:`pypy/interpreter/eval.py` and
:source:`pypy/interpreter/pyframe.py`,
- function objects and argument passing in
:source:`pypy/interpreter/function.py`
- and :source:`pypy/interpreter/argument.py`, the object space interface
- definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
- :source:`pypy/interpreter/module.py` and
:source:`pypy/interpreter/mixedmodule.py`.
- Core types supporting the bytecode interpreter are defined in
:source:`pypy/interpreter/typedef.py`.
-
-* :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
- and grammar files that allow it to parse the syntax of various Python
- versions. Once the grammar has been processed, the parser can be
- translated by the above machinery into efficient code.
-
-* :source:`pypy/interpreter/astcompiler` contains the compiler. This
- contains a modified version of the compiler package from CPython
- that fixes some bugs and is translatable.
-
-* :source:`pypy/objspace/std` contains the :ref:`Standard object space
<standard-object-space>`. The main file
- is :source:`pypy/objspace/std/objspace.py`. For each type, the file
- ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
- as a first approximation. (Some types have multiple implementations.)
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
deleted file mode 100644
--- a/pypy/doc/how-to-contribute.rst
+++ /dev/null
@@ -1,93 +0,0 @@
-How to contribute to PyPy
-=========================
-
-This page describes how to contribute to the PyPy project. The first thing
-to remember is that PyPy project is very different than most projects out
there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
-
-
-Don't just hack
----------------
-
-The first and most important rule how not to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature".
-
-
-Test driven development
------------------------
-
-Instead, we practice a lot of test driven development. This is partly because
-of very high quality requirements for compilers and partly because there is
-simply no other way to get around such complex project, that will keep you
sane.
-There are probably people out there who are smart enough not to need it, we're
-not one of those. You may consider familiarizing yourself with `pytest`_,
-since this is a tool we use for tests.
-This leads to the next issue:
-
-.. _pytest: http://pytest.org/
-
-
-Layers
-------
-
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example
writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
-``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
-After that, you can just compile PyPy and things should just work.
-
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-* **python interpreter** - it's the part implemented in the ``pypy/``
directory.
- It's implemented in RPython, which is a high level static language with
- classes, garbage collection, just-in-time compiler generation and the ability
- to call C. A cool part about it is that it can be run untranslated, so all
- the tests are runnable without translating PyPy.
-
- **interpreter** contains the interpreter core
-
- **objspace** contains implementations of various objects exported to
- the Python layer
-
- **module** directory contains extension modules written in RPython
-
-* **rpython compiler** that resides in ``rpython/annotator`` and
- ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_
- for further reading
-
-* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
- in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
- ``rpython/jit/metainterp`` (runtime part) and
- ``rpython/jit/codewriter`` (translation-time part). Backends live in
- ``rpython/jit/backend``.
-
-* **garbage collection** lives in ``rpython/memory``
-
-The rest of directories serve specific niche goal and are unlikely a good
-entry point.
-
-
-More documentation
-------------------
-
-* `Getting Started Developing With PyPy`_
-
-* `Getting Started with RPython`_
-
-.. _`Getting Started Developing With PyPy`: getting-started-dev.html
-.. _`Getting started with RPython`:
http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -34,6 +34,7 @@
whatsnew-2.0.0-beta1.rst
whatsnew-1.9.rst
+
CPython 3.5 compatible versions
-------------------------------
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -9,7 +9,7 @@
* If you're interested in trying PyPy out, check out the :doc:`installation
instructions <install>`.
-* If you want to help develop PyPy, please have a look at :doc:`how to
contribute <how-to-contribute>`
+* If you want to help develop PyPy, please have a look at :doc:`contributing
<contributing>`
and get in touch (:ref:`contact`)!
All of the documentation and source code is available under the MIT license,
@@ -31,6 +31,7 @@
introduction
install
build
+ windows
faq
@@ -40,43 +41,30 @@
----------
.. toctree::
- :maxdepth: 1
+ :maxdepth: 2
cpython_differences
extending
- embedding
gc_info
jit-hooks
stackless
__pypy__-module
- objspace-proxies
sandbox
stm
- windows
-
-.. _developing-pypy:
-
-Development documentation
--------------------------
+Development
+-----------
.. toctree::
- :maxdepth: 1
+ :maxdepth: 2
- getting-started-dev
- how-to-contribute
- you-want-to-help
+ contributing
architecture
configuration
project-ideas
project-documentation
how-to-release
-.. TODO: audit ^^
-
-
-.. TODO: Fill this in
-
Further resources
-----------------
@@ -84,13 +72,10 @@
.. toctree::
:maxdepth: 1
- extradoc
- eventhistory
- discussions
index-of-release-notes
index-of-whatsnew
contributor
-
+ glossary
.. _contact:
@@ -118,7 +103,7 @@
the `development mailing list`_.
.. _#pypy on irc.freenode.net: irc://irc.freenode.net/pypy
-.. _here: http://www.tismer.com/pypy/irc-logs/pypy/
+.. _here: https://botbot.me/freenode/pypy/
.. _Development mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
.. _Commit mailing list: http://mail.python.org/mailman/listinfo/pypy-commit
.. _Development bug/feature tracker: https://bitbucket.org/pypy/pypy/issues
diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst
--- a/pypy/doc/interpreter.rst
+++ b/pypy/doc/interpreter.rst
@@ -102,7 +102,7 @@
program flows with homogeneous name-value assignments on
function invocations.
-.. _how-to guide for descriptors:
http://users.rcn.com/python/download/Descriptor.htm
+.. _how-to guide for descriptors:
https://docs.python.org/3/howto/descriptor.html
Bytecode Interpreter Implementation Classes
diff --git a/pypy/doc/man/pypy3.1.rst b/pypy/doc/man/pypy3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/man/pypy3.1.rst
@@ -0,0 +1,135 @@
+=======
+ pypy3
+=======
+
+.. note: this is turned into a regular man page "pypy3.1" by
+ doing "make man" in pypy/doc/
+
+SYNOPSIS
+========
+
+``pypy3`` [*options*]
+[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ]
+[*arg*\ ...]
+
+OPTIONS
+=======
+
+-i
+ Inspect interactively after running script.
+
+-O
+ Skip assert statements.
+
+-OO
+ Remove docstrings when importing modules in addition to ``-O``.
+
+-c CMD
+ Program passed in as ``CMD`` (terminates option list).
+
+-S
+ Do not ``import site`` on initialization.
+
+-s
+ Don't add the user site directory to `sys.path`.
+
+-u
+ Unbuffered binary ``stdout`` and ``stderr``.
+
+-h, --help
+ Show a help message and exit.
+
+-m MOD
+ Library module to be run as a script (terminates option list).
+
+-W ARG
+ Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*).
+
+-E
+ Ignore environment variables (such as ``PYTHONPATH``).
+
+-B
+ Disable writing bytecode (``.pyc``) files.
+
+-X track-resources
+ Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+ garbage collector.
+
+--version
+ Print the PyPy version.
+
+--info
+ Print translation information about this PyPy executable.
+
+--jit ARG
+ Low level JIT parameters. Mostly internal. Run ``--jit help``
+ for more information.
+
+ENVIRONMENT
+===========
+
+``PYTHONPATH``
+ Add directories to pypy3's module search path.
+ The format is the same as shell's ``PATH``.
+
+``PYTHONSTARTUP``
+ A script referenced by this variable will be executed before the
+ first prompt is displayed, in interactive mode.
+
+``PYTHONDONTWRITEBYTECODE``
+ If set to a non-empty value, equivalent to the ``-B`` option.
+ Disable writing ``.pyc`` files.
+
+``PYTHONINSPECT``
+ If set to a non-empty value, equivalent to the ``-i`` option.
+ Inspect interactively after running the specified script.
+
+``PYTHONIOENCODING``
+ If this is set, it overrides the encoding used for
+ *stdin*/*stdout*/*stderr*.
+ The syntax is *encodingname*:*errorhandler*
+ The *errorhandler* part is optional and has the same meaning as in
+ `str.encode`.
+
+``PYTHONNOUSERSITE``
+ If set to a non-empty value, equivalent to the ``-s`` option.
+ Don't add the user site directory to `sys.path`.
+
+``PYTHONWARNINGS``
+ If set, equivalent to the ``-W`` option (warning control).
+ The value should be a comma-separated list of ``-W`` parameters.
+
+``PYPYLOG``
+ If set to a non-empty value, enable logging, the format is:
+
+ *fname* or *+fname*
+ logging for profiling: includes all
+ ``debug_start``/``debug_stop`` but not any nested
+ ``debug_print``.
+ *fname* can be ``-`` to log to *stderr*.
+ The *+fname* form can be used if there is a *:* in fname
+
+ ``:``\ *fname*
+ Full logging, including ``debug_print``.
+
+ *prefix*\ ``:``\ *fname*
+ Conditional logging.
+ Multiple prefixes can be specified, comma-separated.
+ Only sections whose name match the prefix will be logged.
+
+ ``PYPYLOG=jit-log-opt,jit-backend:logfile`` will
+ generate a log suitable for *jitviewer*, a tool for debugging
+ performance issues under PyPy.
+
+``PYPY_IRC_TOPIC``
+ If set to a non-empty value, print a random #pypy IRC
+ topic at startup of interactive mode.
+
+
+.. include:: ../gc_info.rst
+ :start-line: 7
+
+SEE ALSO
+========
+
+**python3**\ (1)
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -1,28 +1,7 @@
-What PyPy can do for your objects
-=================================
-
-.. contents::
-
-
-Thanks to the :doc:`Object Space <objspace>` architecture, any feature that is
-based on proxying, extending, changing or otherwise controlling the
-behavior of objects in a running program is easy to implement on top of PyPy.
-
-Here is what we have implemented so far, in historical order:
-
-* *Dump Object Space*: dumps all operations performed on all the objects
- into a large log file. For debugging your applications.
-
-* *Transparent Proxies Extension*: adds new proxy objects to
- the Standard Object Space that enable applications to
- control operations on application and builtin objects,
- e.g lists, dictionaries, tracebacks.
-
-
.. _tproxy:
-Transparent Proxies
--------------------
+Transparent Proxies (DEPRECATED)
+--------------------------------
.. warning::
@@ -194,7 +173,7 @@
application-level code.
Transparent proxies are implemented on top of the :ref:`standard object
-space <standard-object-space>`, in
:source:`pypy/objspace/std/proxy_helpers.py`,
+space <standard-object-space>`, in :source:`pypy/objspace/std/proxyobject.py`,
:source:`pypy/objspace/std/proxyobject.py` and
:source:`pypy/objspace/std/transparent.py`.
To use them you will need to pass a `--objspace-std-withtproxy`_ option to
``pypy``
or ``translate.py``. This registers implementations named
:py:class:`W_TransparentXxx`
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -474,8 +474,8 @@
:source:`pypy/objspace/std/bytesobject.py` defines ``W_AbstractBytesObject``,
which contains everything needed to build the ``str`` app-level type;
and there are subclasses ``W_BytesObject`` (the usual string) and
-``W_StringBufferObject`` (a special implementation tweaked for repeated
-additions, in :source:`pypy/objspace/std/strbufobject.py`). For mutable data
+``W_Buffer`` (a special implementation tweaked for repeated
+additions, in :source:`pypy/objspace/std/bufferobject.py`). For mutable data
types like lists and dictionaries, we have a single class
``W_ListObject`` or ``W_DictMultiObject`` which has an indirection to
the real data and a strategy; the strategy can change as the content of
diff --git a/pypy/doc/project-documentation.rst
b/pypy/doc/project-documentation.rst
--- a/pypy/doc/project-documentation.rst
+++ b/pypy/doc/project-documentation.rst
@@ -32,10 +32,13 @@
coding-guide
sprint-reports
extradoc
+ eventhistory
video-index
index-report
+ discussions
dev_method
- glossary
+ embedding
+ objspace-proxies
Source Code Documentation
diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst
--- a/pypy/doc/release-v6.0.0.rst
+++ b/pypy/doc/release-v6.0.0.rst
@@ -8,13 +8,18 @@
the dual release.
This release is a feature release following our previous 5.10 incremental
-release in late December 2017. Our C-API compatability layer ``cpyext`` is
+release in late December 2017. Our C-API compatibility layer ``cpyext`` is
now much faster (see the `blog post`_) as well as more complete. We have made
many other improvements in speed and CPython compatibility. Since the changes
affect the included python development header files, all c-extension modules
must
be recompiled for this version.
-First-time python users are often stumped by silly typos and emissions when
+Until we can work with downstream providers to distribute builds with PyPy, we
+have made packages for some common packages `available as wheels`_. You may
+compile yourself using ``pip install --no-build-isolation <package>``, the
+``no-build-isolation`` is currently needed for pip v10.
+
+First-time python users are often stumped by silly typos and omissions when
getting started writing code. We have improved our parser to emit more friendly
`syntax errors`_, making PyPy not only faster but more friendly.
@@ -60,6 +65,7 @@
.. _`hooks`: gc_info.html#gc-hooks
.. _`cffi`: http://cffi.readthedocs.io
.. _`cppyy`: https://cppyy.readthedocs.io
+.. _`available as wheels`: https://github.com/antocuni/pypy-wheels
What is PyPy?
=============
@@ -110,7 +116,7 @@
* Fix JIT bugs exposed in the sre module
* Improve speed of Python parser, improve ParseError messages and SyntaxError
* Handle JIT hooks more efficiently
-* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s
+* Fix a rare GC bug exposed by intensive use of cpyext ``Buffer`` s
We also refactored many parts of the JIT bridge optimizations, as well as
cpyext
internals, and together with new contributors fixed issues, added new
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,5 +5,20 @@
.. this is a revision shortly after release-pypy-6.0.0
.. startrev: e50e11af23f1
+.. branch: cppyy-packaging
+Upgrade to backend 0.6.0, support exception handling from wrapped functions,
+update enum handling, const correctness for data members and associated tests,
+support anonymous enums, support for function pointer arguments
+.. branch: socket_default_timeout_blockingness
+
+Make sure 'blocking-ness' of socket is set along with default timeout
+
+.. branch: crypt_h
+
+Include crypt.h for crypt() on Linux
+
+.. branch: gc-more-logging
+
+Log additional gc-minor and gc-collect-step info in the PYPYLOG
diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst
b/pypy/doc/whatsnew-pypy2-5.10.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.10.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst
@@ -32,6 +32,7 @@
.. branch: fix-vmprof-stacklet-switch
.. branch: fix-vmprof-stacklet-switch-2
+
Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...)
.. branch: win32-vcvars
@@ -39,4 +40,3 @@
.. branch: rdict-fast-hash
Make it possible to declare that the hash function of an r_dict is fast in
RPython.
-
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -25,7 +25,7 @@
This compiler, while the standard one for Python 2.7, is deprecated. Microsoft
has
made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the
link
-was checked in Nov 2016). Note that the compiler suite may be installed in
+was checked in May 2018). Note that the compiler suite may be installed in
``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for
Python``
or in
``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``.
@@ -37,7 +37,7 @@
by running the mt.exe command by hand from a DOS window (that is how the author
discovered the problem).
-.. _Microsoft Visual C++ Compiler for Python 2.7:
https://www.microsoft.com/en-us/download/details.aspx?id=44266
+.. _Microsoft Visual C++ Compiler for Python 2.7:
https://www.microsoft.com/EN-US/DOWNLOAD/DETAILS.ASPX?ID=44266
Installing "Build Tools for Visual Studio 2017" (for Python 3)
--------------------------------------------------------------
@@ -150,7 +150,7 @@
build dependencies for windows. As part of the `rpython` setup stage,
environment
variables will be set to use these dependencies. The repository has a README
file on how to replicate, and a branch for each supported platform. You may run
- the `get_externals.py` utility to checkout the proper branch for your platform
+the `get_externals.py` utility to checkout the proper branch for your platform
and PyPy version.
.. _repository: https://bitbucket.org/pypy/external
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
deleted file mode 100644
--- a/pypy/doc/you-want-to-help.rst
+++ /dev/null
@@ -1,81 +0,0 @@
-You want to help with PyPy, now what?
-=====================================
-
-PyPy is a very large project that has a reputation of being hard to dive into.
-Some of this fame is warranted, some of it is purely accidental. There are
three
-important lessons that everyone willing to contribute should learn:
-
-* PyPy has layers. There are many pieces of architecture that are very well
- separated from each other. More about this below, but often the manifestation
- of this is that things are at a different layer than you would expect them
- to be. For example if you are looking for the JIT implementation, you will
- not find it in the implementation of the Python programming language.
-
-* Because of the above, we are very serious about Test Driven Development.
- It's not only what we believe in, but also that PyPy's architecture is
- working very well with TDD in mind and not so well without it. Often
- development means progressing in an unrelated corner, one unittest
- at a time; and then flipping a giant switch, bringing it all together.
- (It generally works out of the box. If it doesn't, then we didn't
- write enough unit tests.) It's worth repeating - PyPy's
- approach is great if you do TDD, and not so great otherwise.
-
-* PyPy uses an entirely different set of tools - most of them included
- in the PyPy repository. There is no Makefile, nor autoconf. More below.
-
-
-Architecture
-------------
-
-PyPy has layers. The 100 miles view:
-
-* :ref:`RPython <rpython:language>` is the language in which we write
interpreters. Not the entire
- PyPy project is written in RPython, only the parts that are compiled in
- the translation process. The interesting point is that RPython has no parser,
- it's compiled from the live python objects, which makes it possible to do
- all kinds of metaprogramming during import time. In short, Python is a meta
- programming language for RPython.
-
- The RPython standard library is to be found in the ``rlib`` subdirectory.
-
-* The translation toolchain - this is the part that takes care of translating
- RPython to flow graphs and then to C. There is more in the
:doc:`architecture <architecture>`
- document written about it.
-
- It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
- and ``rtyper``.
-
-* Python Interpreter and modules
-
- This is in the ``pypy`` directory. ``pypy/interpreter`` is a standard
- interpreter for Python written in RPython. The fact that it is
- RPython is not apparent at first. Built-in modules are written in
- ``pypy/module/*``. Some modules that CPython implements in C are
- simply written in pure Python; they are in the top-level ``lib_pypy``
- directory. The standard library of Python (with a few changes to
- accomodate PyPy) is in ``lib-python``.
-
-* :ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that
traces the
- interpreter written in RPython, rather than the user program that it
- interprets. As a result it applies to any interpreter, i.e. any
- language. But getting it to work correctly is not trivial: it
- requires a small number of precise "hints" and possibly some small
- refactorings of the interpreter. The JIT itself also has several
- almost-independent parts: the tracer itself in ``rpython/jit/metainterp``,
the
- optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
- residual operations, and the backend in
``rpython/jit/backend/<machine-name>``
- that turns it into machine code. Writing a new backend is a
- traditional way to get into the project.
-
-* Garbage Collectors (GC): as you may notice if you are used to CPython's
- C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
- :ref:`rpython:garbage-collection` is inserted
- during translation. Moreover, this is not reference counting; it is a real
- GC written as more RPython code. The best one we have so far is in
- ``rpython/memory/gc/incminimark.py``.
-
-
-Toolset
--------
-
-xxx
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -499,6 +499,7 @@
def visit_truncatedint_w(self, typ):
self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
+ @staticmethod
def make_fastfunc(unwrap_spec, func):
unwrap_info = UnwrapSpec_FastFunc_Unwrap()
unwrap_info.apply_over(unwrap_spec)
@@ -529,7 +530,6 @@
exec compile2(source) in unwrap_info.miniglobals, d
fastfunc = d['fastfunc_%s_%d' % (func.__name__.replace('-', '_'),
narg)]
return narg, fastfunc
- make_fastfunc = staticmethod(make_fastfunc)
def int_unwrapping_space_method(typ):
diff --git a/pypy/interpreter/unicodehelper.py
b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -78,6 +78,216 @@
allow_surrogates=True)
# ____________________________________________________________
+# utf-16
+
+def str_decode_utf_16(s, size, errors, final=True,
+ errorhandler=None):
+ result, length, byteorder = str_decode_utf_16_helper(s, size, errors,
final,
+ errorhandler,
"native")
+ return result, length
+
+def str_decode_utf_16_be(s, size, errors, final=True,
+ errorhandler=None):
+ result, length, byteorder = str_decode_utf_16_helper(s, size, errors,
final,
+ errorhandler, "big")
+ return result, length
+
+def str_decode_utf_16_le(s, size, errors, final=True,
+ errorhandler=None):
+ result, length, byteorder = str_decode_utf_16_helper(s, size, errors,
final,
+ errorhandler,
"little")
+ return result, length
+
+def str_decode_utf_16_helper(s, size, errors, final=True,
+ errorhandler=None,
+ byteorder="native",
+ public_encoding_name='utf16'):
+ if errorhandler is None:
+ errorhandler = default_unicode_error_decode
+ bo = 0
+
+ if BYTEORDER == 'little':
+ ihi = 1
+ ilo = 0
+ else:
+ ihi = 0
+ ilo = 1
+
+ # Check for BOM marks (U+FEFF) in the input and adjust current
+ # byte order setting accordingly. In native mode, the leading BOM
+ # mark is skipped, in all other modes, it is copied to the output
+ # stream as-is (giving a ZWNBSP character).
+ pos = 0
+ if byteorder == 'native':
+ if size >= 2:
+ bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+ if BYTEORDER == 'little':
+ if bom == 0xFEFF:
+ pos += 2
+ bo = -1
+ elif bom == 0xFFFE:
+ pos += 2
+ bo = 1
+ else:
+ if bom == 0xFEFF:
+ pos += 2
+ bo = 1
+ elif bom == 0xFFFE:
+ pos += 2
+ bo = -1
+ elif byteorder == 'little':
+ bo = -1
+ else:
+ bo = 1
+ if size == 0:
+ return u'', 0, bo
+ if bo == -1:
+ # force little endian
+ ihi = 1
+ ilo = 0
+
+ elif bo == 1:
+ # force big endian
+ ihi = 0
+ ilo = 1
+
+ result = UnicodeBuilder(size // 2)
+
+ #XXX I think the errors are not correctly handled here
+ while pos < size:
+ # remaining bytes at the end? (size should be even)
+ if len(s) - pos < 2:
+ if not final:
+ break
+ r, pos = errorhandler(errors, public_encoding_name,
+ "truncated data",
+ s, pos, len(s))
+ result.append(r)
+ if len(s) - pos < 2:
+ break
+ ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo])
+ pos += 2
+ if ch < 0xD800 or ch > 0xDFFF:
+ result.append(unichr(ch))
+ continue
+ # UTF-16 code pair:
+ if len(s) - pos < 2:
+ pos -= 2
+ if not final:
+ break
+ errmsg = "unexpected end of data"
+ r, pos = errorhandler(errors, public_encoding_name,
+ errmsg, s, pos, len(s))
+ result.append(r)
+ if len(s) - pos < 2:
+ break
+ elif 0xD800 <= ch <= 0xDBFF:
+ ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo])
+ pos += 2
+ if 0xDC00 <= ch2 <= 0xDFFF:
+ if MAXUNICODE < 65536:
+ result.append(unichr(ch))
+ result.append(unichr(ch2))
+ else:
+ result.append(UNICHR((((ch & 0x3FF)<<10) |
+ (ch2 & 0x3FF)) + 0x10000))
+ continue
+ else:
+ r, pos = errorhandler(errors, public_encoding_name,
+ "illegal UTF-16 surrogate",
+ s, pos - 4, pos - 2)
+ result.append(r)
+ else:
+ r, pos = errorhandler(errors, public_encoding_name,
+ "illegal encoding",
+ s, pos - 2, pos)
+ result.append(r)
+ return result.build(), pos, bo
+
+def _STORECHAR(result, CH, byteorder):
+ hi = chr(((CH) >> 8) & 0xff)
+ lo = chr((CH) & 0xff)
+ if byteorder == 'little':
+ result.append(lo)
+ result.append(hi)
+ else:
+ result.append(hi)
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
