Author: fijal Branch: jit-leaner-frontend Changeset: r82635:3d2c4e4fc169 Date: 2016-03-01 14:04 +0100 http://bitbucket.org/pypy/pypy/changeset/3d2c4e4fc169/
Log: merge default diff too long, truncating to 2000 out of 12385 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -22,6 +22,7 @@ ^pypy/module/cpyext/test/.+\.obj$ ^pypy/module/cpyext/test/.+\.manifest$ ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$ +^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$ ^pypy/module/cppyy/src/.+\.o$ ^pypy/module/cppyy/bench/.+\.so$ ^pypy/module/cppyy/bench/.+\.root$ @@ -35,7 +36,6 @@ ^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$ ^pypy/doc/.+\.html$ ^pypy/doc/config/.+\.rst$ -^pypy/doc/basicblock\.asc$ ^pypy/doc/.+\.svninfo$ ^rpython/translator/c/src/libffi_msvc/.+\.obj$ ^rpython/translator/c/src/libffi_msvc/.+\.dll$ @@ -45,53 +45,33 @@ ^rpython/translator/c/src/cjkcodecs/.+\.obj$ ^rpython/translator/c/src/stacklet/.+\.o$ ^rpython/translator/c/src/.+\.o$ -^rpython/translator/jvm/\.project$ -^rpython/translator/jvm/\.classpath$ -^rpython/translator/jvm/eclipse-bin$ -^rpython/translator/jvm/src/pypy/.+\.class$ -^rpython/translator/benchmark/docutils$ -^rpython/translator/benchmark/templess$ -^rpython/translator/benchmark/gadfly$ -^rpython/translator/benchmark/mako$ -^rpython/translator/benchmark/bench-custom\.benchmark_result$ -^rpython/translator/benchmark/shootout_benchmarks$ +^rpython/translator/llvm/.+\.so$ ^rpython/translator/goal/target.+-c$ ^rpython/translator/goal/.+\.exe$ ^rpython/translator/goal/.+\.dll$ ^pypy/goal/pypy-translation-snapshot$ ^pypy/goal/pypy-c -^pypy/goal/pypy-jvm -^pypy/goal/pypy-jvm.jar ^pypy/goal/.+\.exe$ ^pypy/goal/.+\.dll$ ^pypy/goal/.+\.lib$ ^pypy/_cache$ -^pypy/doc/statistic/.+\.html$ -^pypy/doc/statistic/.+\.eps$ -^pypy/doc/statistic/.+\.pdf$ -^rpython/translator/cli/src/pypylib\.dll$ -^rpython/translator/cli/src/query\.exe$ -^rpython/translator/cli/src/main\.exe$ +^lib-python/2.7/lib2to3/.+\.pickle$ ^lib_pypy/__pycache__$ ^lib_pypy/ctypes_config_cache/_.+_cache\.py$ ^lib_pypy/ctypes_config_cache/_.+_.+_\.py$ ^lib_pypy/_libmpdec/.+.o$ -^rpython/translator/cli/query-descriptions$ ^pypy/doc/discussion/.+\.html$ ^include/.+\.h$ ^include/.+\.inl$ ^pypy/doc/_build/.*$ ^pypy/doc/config/.+\.html$ ^pypy/doc/config/style\.css$ -^pypy/doc/jit/.+\.html$ -^pypy/doc/jit/style\.css$ ^pypy/doc/image/lattice1\.png$ ^pypy/doc/image/lattice2\.png$ ^pypy/doc/image/lattice3\.png$ ^pypy/doc/image/stackless_informal\.png$ ^pypy/doc/image/parsing_example.+\.png$ ^rpython/doc/_build/.*$ -^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$ ^compiled ^.git/ ^release/ diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -41,29 +41,29 @@ Amaury Forgeot d'Arc Antonio Cuni Samuele Pedroni + Matti Picus Alex Gaynor Brian Kearns - Matti Picus Philip Jenvey Michael Hudson + Ronan Lamy David Schneider + Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo - Manuel Jacob - Ronan Lamy Benjamin Peterson + Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen - Richard Plangger Richard Emslie Alexander Schremmer Dan Villiom Podlaski Christiansen + Remi Meier Lukas Diekmann Sven Hager Anders Lehmann - Remi Meier Aurelien Campeas Niklaus Haldimann Camillo Bruni @@ -72,8 +72,8 @@ Romain Guillebert Leonardo Santagada Seo Sanghyeon + Ronny Pfannschmidt Justin Peel - Ronny Pfannschmidt David Edelsohn Anders Hammarquist Jakub Gustak @@ -95,6 +95,7 @@ Tyler Wade Michael Foord Stephan Diehl + Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -105,9 +106,9 @@ Jean-Paul Calderone Timo Paulssen Squeaky + Marius Gedminas Alexandre Fayolle Simon Burton - Marius Gedminas Martin Matusiak Konstantin Lopuhin Wenzhu Man @@ -116,16 +117,20 @@ Ivan Sichmann Freitas Greg Price Dario Bertini + Stefano Rivera Mark Pearse Simon Cross Andreas Stührk - Stefano Rivera + Edd Barrett Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov + Jeremy Thurgood Paweł Piotr Przeradowski + Spenser Bauman Paul deGrandis Ilya Osadchiy + marky1991 Tobias Oberstein Adrian Kuhn Boris Feigin @@ -134,14 +139,12 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Edd Barrett + Tobias Pape Wanja Saatkamp Gerald Klix Mike Blume - Tobias Pape Oscar Nierstrasz Stefan H. Muller - Jeremy Thurgood Rami Chowdhury Eugene Oden Henry Mason @@ -153,6 +156,8 @@ Lukas Renggli Guenter Jantzen Ned Batchelder + Tim Felgentreff + Anton Gulenko Amit Regmi Ben Young Nicolas Chauvat @@ -162,12 +167,12 @@ Nicholas Riley Jason Chu Igor Trindade Oliveira - Tim Felgentreff + Yichao Yu Rocco Moretti Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila - Yichao Yu + Devin Jeanpierre Gabriel Lavoie Olivier Dormond Jared Grubb @@ -191,33 +196,33 @@ Stanislaw Halik Mikael Schönenberg Berkin Ilbeyi - Elmo M?ntynen + Elmo Mäntynen + Faye Zhao Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani Beatrice During Alex Perry - Vincent Legoll + Vaibhav Sood Alan McIntyre - Spenser Bauman + William Leslie Alexander Sedov Attila Gobi + Jasper.Schulz Christopher Pope - Devin Jeanpierre - Vaibhav Sood Christian Tismer Marc Abramowitz Dan Stromberg Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + Mark Young Alexis Daboville Jens-Uwe Mager Carl Meyer Karl Ramm Pieter Zieschang - Anton Gulenko Gabriel Lukas Vacek Andrew Dalke @@ -225,6 +230,7 @@ Jakub Stasiak Nathan Taylor Vladimir Kryachko + Omer Katz Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -239,6 +245,7 @@ Lars Wassermann Philipp Rustemeuer Henrik Vendelbo + Richard Lancaster Dan Buch Miguel de Val Borro Artur Lisiecki @@ -250,18 +257,18 @@ Tomo Cocoa Kim Jin Su Toni Mattis + Amber Brown Lucas Stadler Julian Berman Markus Holtermann roberto@goyle Yury V. Zaytsev Anna Katrina Dominguez - William Leslie Bobby Impollonia - Faye Zhao [email protected] Andrew Thompson Yusei Tahara + Aaron Tubbs Ben Darnell Roberto De Ioris Juan Francisco Cantero Hurtado @@ -273,6 +280,7 @@ Christopher Armstrong Michael Hudson-Doyle Anders Sigfridsson + Nikolay Zinov Yasir Suhail Jason Michalski [email protected] @@ -282,6 +290,7 @@ Gustavo Niemeyer Stephan Busemann Rafał Gałczyński + Matt Bogosian Christian Muirhead Berker Peksag James Lan @@ -316,9 +325,9 @@ Stefan Marr jiaaro Mads Kiilerich - Richard Lancaster opassembler.py Antony Lee + Jason Madden Yaroslav Fedevych Jim Hunziker Markus Unterwaditzer @@ -327,6 +336,7 @@ squeaky Zearin soareschen + Jonas Pfannschmidt Kurt Griffiths Mike Bayer Matthew Miller diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -188,7 +188,7 @@ # the 'libs' directory is for binary installs - we assume that # must be the *native* platform. But we don't really support # cross-compiling via a binary install anyway, so we let it go. - self.library_dirs.append(os.path.join(sys.exec_prefix, 'include')) + self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -7,6 +7,7 @@ content = fid.read() # from cffi's Verifier() key = '\x00'.join([sys.version[:3], content]) + key += 'cpyext-gc-support-2' # this branch requires recompilation! if sys.version_info >= (3,): key = key.encode('utf-8') k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) @@ -62,7 +63,7 @@ if sys.platform == 'win32': # XXX pyconfig.h uses a pragma to link to the import library, # which is currently python27.lib - library = os.path.join(thisdir, '..', 'include', 'python27') + library = os.path.join(thisdir, '..', 'libs', 'python27') if not os.path.exists(library + '.lib'): # For a local translation or nightly build library = os.path.join(thisdir, '..', 'pypy', 'goal', 'python27') diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -550,21 +550,24 @@ lst.append(value) # if '__pypy__' in sys.builtin_module_names: + import os if sys.platform == "win32": - # we need 'libpypy-c.lib'. Right now, distributions of - # pypy contain it as 'include/python27.lib'. You need - # to manually copy it back to 'libpypy-c.lib'. XXX Will - # be fixed in the next pypy release. - pythonlib = "libpypy-c" + # we need 'libpypy-c.lib'. Current distributions of + # pypy (>= 4.1) contain it as 'libs/python27.lib'. + pythonlib = "python27" if hasattr(sys, 'prefix'): - ensure('library_dirs', sys.prefix) + ensure('library_dirs', os.path.join(sys.prefix, 'libs')) else: # we need 'libpypy-c.{so,dylib}', which should be by - # default located in 'sys.prefix/bin' + # default located in 'sys.prefix/bin' for installed + # systems. pythonlib = "pypy-c" if hasattr(sys, 'prefix'): - import os ensure('library_dirs', os.path.join(sys.prefix, 'bin')) + # On uninstalled pypy's, the libpypy-c is typically found in + # .../pypy/goal/. + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'pypy', 'goal')) else: if sys.platform == "win32": template = "python%d%d" diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -36,9 +36,15 @@ "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend", - "_csv", "cppyy", "_pypyjson", "_vmprof", + "_csv", "cppyy", "_pypyjson", ]) +from rpython.jit.backend import detect_cpu +try: + if detect_cpu.autodetect().startswith('x86'): + working_modules.add('_vmprof') +except detect_cpu.ProcessorAutodetectError: + pass translation_modules = default_modules.copy() translation_modules.update([ @@ -163,12 +169,8 @@ cmdline="--translationmodules", suggests=[("objspace.allworkingmodules", False)]), - BoolOption("usepycfiles", "Write and read pyc files when importing", - default=True), - BoolOption("lonepycfiles", "Import pyc files with no matching py file", - default=False, - requires=[("objspace.usepycfiles", True)]), + default=False), StrOption("soabi", "Tag to differentiate extension modules built for different Python interpreters", diff --git a/pypy/doc/discussion/rawrefcount.rst b/pypy/doc/discussion/rawrefcount.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/discussion/rawrefcount.rst @@ -0,0 +1,158 @@ +====================== +Rawrefcount and the GC +====================== + + +GC Interface +------------ + +"PyObject" is a raw structure with at least two fields, ob_refcnt and +ob_pypy_link. The ob_refcnt is the reference counter as used on +CPython. If the PyObject structure is linked to a live PyPy object, +its current address is stored in ob_pypy_link and ob_refcnt is bumped +by either the constant REFCNT_FROM_PYPY, or the constant +REFCNT_FROM_PYPY_LIGHT (== REFCNT_FROM_PYPY + SOME_HUGE_VALUE) +(to mean "light finalizer"). + +Most PyPy objects exist outside cpyext, and conversely in cpyext it is +possible that a lot of PyObjects exist without being seen by the rest +of PyPy. At the interface, however, we can "link" a PyPy object and a +PyObject. There are two kinds of link: + +rawrefcount.create_link_pypy(p, ob) + + Makes a link between an exising object gcref 'p' and a newly + allocated PyObject structure 'ob'. ob->ob_refcnt must be + initialized to either REFCNT_FROM_PYPY, or + REFCNT_FROM_PYPY_LIGHT. (The second case is an optimization: + when the GC finds the PyPy object and PyObject no longer + referenced, it can just free() the PyObject.) + +rawrefcount.create_link_pyobj(p, ob) + + Makes a link from an existing PyObject structure 'ob' to a newly + allocated W_CPyExtPlaceHolderObject 'p'. You must also add + REFCNT_FROM_PYPY to ob->ob_refcnt. For cases where the PyObject + contains all the data, and the PyPy object is just a proxy. The + W_CPyExtPlaceHolderObject should have only a field that contains + the address of the PyObject, but that's outside the scope of the + GC. + +rawrefcount.from_obj(p) + + If there is a link from object 'p' made with create_link_pypy(), + returns the corresponding 'ob'. Otherwise, returns NULL. + +rawrefcount.to_obj(Class, ob) + + Returns ob->ob_pypy_link, cast to an instance of 'Class'. + + +Collection logic +---------------- + +Objects existing purely on the C side have ob->ob_pypy_link == 0; +these are purely reference counted. On the other hand, if +ob->ob_pypy_link != 0, then ob->ob_refcnt is at least REFCNT_FROM_PYPY +and the object is part of a "link". + +The idea is that links whose 'p' is not reachable from other PyPy +objects *and* whose 'ob->ob_refcnt' is REFCNT_FROM_PYPY or +REFCNT_FROM_PYPY_LIGHT are the ones who die. But it is more messy +because PyObjects still (usually) need to have a tp_dealloc called, +and this cannot occur immediately (and can do random things like +accessing other references this object points to, or resurrecting the +object). + +Let P = list of links created with rawrefcount.create_link_pypy() +and O = list of links created with rawrefcount.create_link_pyobj(). +The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all +the data is in the PyObjects, and all outsite references (if any) are +in C, as "PyObject *" fields. + +So, during the collection we do this about P links: + + for (p, ob) in P: + if ob->ob_refcnt != REFCNT_FROM_PYPY + and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT: + mark 'p' as surviving, as well as all its dependencies + +At the end of the collection, the P and O links are both handled like +this: + + for (p, ob) in P + O: + if p is not surviving: # even if 'ob' might be surviving + unlink p and ob + if ob->ob_refcnt == REFCNT_FROM_PYPY_LIGHT: + free(ob) + elif ob->ob_refcnt > REFCNT_FROM_PYPY_LIGHT: + ob->ob_refcnt -= REFCNT_FROM_PYPY_LIGHT + else: + ob->ob_refcnt -= REFCNT_FROM_PYPY + if ob->ob_refcnt == 0: + invoke _Py_Dealloc(ob) later, outside the GC + + +GC Implementation +----------------- + +We need two copies of both the P list and O list, for young or old +objects. All four lists can be regular AddressLists of 'ob' objects. + +We also need an AddressDict mapping 'p' to 'ob' for all links in the P +list, and update it when PyPy objects move. + + +Further notes +------------- + +XXX +XXX the rest is the ideal world, but as a first step, we'll look +XXX for the minimal tweaks needed to adapt the existing cpyext +XXX + +For objects that are opaque in CPython, like <dict>, we always create +a PyPy object, and then when needed we make an empty PyObject and +attach it with create_link_pypy()/REFCNT_FROM_PYPY_LIGHT. + +For <int> and <float> objects, the corresponding PyObjects contain a +"long" or "double" field too. We link them with create_link_pypy() +and we can use REFCNT_FROM_PYPY_LIGHT too: 'tp_dealloc' doesn't +need to be called, and instead just calling free() is fine. + +For <type> objects, we need both a PyPy and a PyObject side. These +are made with create_link_pypy()/REFCNT_FROM_PYPY. + +For custom PyXxxObjects allocated from the C extension module, we +need create_link_pyobj(). + +For <str> or <unicode> objects coming from PyPy, we use +create_link_pypy()/REFCNT_FROM_PYPY_LIGHT with a PyObject +preallocated with the size of the string. We copy the string +lazily into that area if PyString_AS_STRING() is called. + +For <str>, <unicode>, <tuple> or <list> objects in the C extension +module, we first allocate it as only a PyObject, which supports +mutation of the data from C, like CPython. When it is exported to +PyPy we could make a W_CPyExtPlaceHolderObject with +create_link_pyobj(). + +For <tuple> objects coming from PyPy, if they are not specialized, +then the PyPy side holds a regular reference to the items. Then we +can allocate a PyTupleObject and store in it borrowed PyObject +pointers to the items. Such a case is created with +create_link_pypy()/REFCNT_FROM_PYPY_LIGHT. If it is specialized, +then it doesn't work because the items are created just-in-time on the +PyPy side. In this case, the PyTupleObject needs to hold real +references to the PyObject items, and we use create_link_pypy()/ +REFCNT_FROM_PYPY. In all cases, we have a C array of PyObjects +that we can directly return from PySequence_Fast_ITEMS, PyTuple_ITEMS, +PyTuple_GetItem, and so on. + +For <list> objects coming from PyPy, we can use a cpyext list +strategy. The list turns into a PyListObject, as if it had been +allocated from C in the first place. The special strategy can hold +(only) a direct reference to the PyListObject, and we can use either +create_link_pyobj() or create_link_pypy() (to be decided). +PySequence_Fast_ITEMS then works for lists too, and PyList_GetItem +can return a borrowed reference, and so on. diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -1,5 +1,20 @@ -Making a PyPy Release -===================== +The PyPy Release Process +======================== + +Release Policy +++++++++++++++ + +We try to create a stable release a few times a year. These are released on +a branch named like release-2.x or release-4.x, and each release is tagged, +for instance release-4.0.1. + +After release, inevitably there are bug fixes. It is the responsibility of +the commiter who fixes a bug to make sure this fix is on the release branch, +so that we can then create a tagged bug-fix release, which will hopefully +happen more often than stable releases. + +How to Create a PyPy Release +++++++++++++++++++++++++++++ Overview -------- diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -72,6 +72,7 @@ 'Anton Gulenko':['anton gulenko', 'anton_gulenko'], 'Richard Lancaster':['richardlancaster'], 'William Leslie':['William ML Leslie'], + 'Spenser Bauman':['Spenser Andrew Bauman'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -128,6 +128,7 @@ Fix SSL tests by importing cpython's patch + .. branch: remove-getfield-pure Remove pure variants of ``getfield_gc_*`` operations from the JIT. Relevant @@ -152,3 +153,34 @@ Seperate structmember.h from Python.h Also enhance creating api functions to specify which header file they appear in (previously only pypy_decl.h) + +.. branch: llimpl + +Refactor register_external(), remove running_on_llinterp mechanism and +apply sandbox transform on externals at the end of annotation. + +.. branch: cffi-embedding-win32 + +.. branch: windows-vmprof-support + +vmprof should work on Windows. + + +.. branch: reorder-map-attributes + +When creating instances and adding attributes in several different orders +depending on some condition, the JIT would create too much code. This is now +fixed. + +.. branch: cpyext-gc-support-2 + +Improve CPython C API support, which means lxml now runs unmodified +(after removing pypy hacks, pending pull request) + +.. branch: look-inside-tuple-hash + +Look inside tuple hash, improving mdp benchmark + +.. branch: vlen-resume + +Compress resume data, saving 10-20% of memory consumed by the JIT \ No newline at end of file diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -239,6 +239,9 @@ raise Exception("Cannot use the --output option with PyPy " "when --shared is on (it is by default). " "See issue #1971.") + if sys.platform == 'win32': + config.translation.libname = '..\\..\\libs\\python27.lib' + thisdir.join('..', '..', 'libs').ensure(dir=1) if config.translation.thread: config.objspace.usemodules.thread = True @@ -274,7 +277,6 @@ if config.translation.sandbox: config.objspace.lonepycfiles = False - config.objspace.usepycfiles = False config.translating = True diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -27,7 +27,7 @@ class W_Root(object): """This is the abstract root class of all wrapped objects that live in a 'normal' object space like StdObjSpace.""" - __slots__ = () + __slots__ = ('__weakref__',) user_overridden_class = False def getdict(self, space): diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -91,6 +91,7 @@ strstart = (0, 0, "") for line in lines: lnum = lnum + 1 + line = universal_newline(line) pos, max = 0, len(line) if contstr: @@ -259,3 +260,14 @@ token_list.append((tokens.ENDMARKER, '', lnum, pos, line)) return token_list + + +def universal_newline(line): + # show annotator that indexes below are non-negative + line_len_m2 = len(line) - 2 + if line_len_m2 >= 0 and line[-2] == '\r' and line[-1] == '\n': + return line[:line_len_m2] + '\n' + line_len_m1 = len(line) - 1 + if line_len_m1 >= 0 and line[-1] == '\r': + return line[:line_len_m1] + '\n' + return line diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -158,3 +158,10 @@ def test_print_function(self): self.parse("from __future__ import print_function\nx = print\n") + + def test_universal_newlines(self): + fmt = 'stuff = """hello%sworld"""' + expected_tree = self.parse(fmt % '\n') + for linefeed in ["\r\n","\r"]: + tree = self.parse(fmt % linefeed) + assert expected_tree == tree diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -156,20 +156,6 @@ get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo" _subclass_cache = {} -def enum_interplevel_subclasses(config, cls): - """Return a list of all the extra interp-level subclasses of 'cls' that - can be built by get_unique_interplevel_subclass().""" - result = [] - for flag1 in (False, True): - for flag2 in (False, True): - for flag3 in (False, True): - for flag4 in (False, True): - result.append(get_unique_interplevel_subclass( - config, cls, flag1, flag2, flag3, flag4)) - result = dict.fromkeys(result) - assert len(result) <= 6 - return result.keys() - def _getusercls(config, cls, wants_dict, wants_slots, wants_del, weakrefable): typedef = cls.typedef if wants_dict and typedef.hasdict: @@ -262,7 +248,7 @@ def user_setup(self, space, w_subtype): self.space = space self.w__class__ = w_subtype - self.user_setup_slots(w_subtype.nslots) + self.user_setup_slots(w_subtype.layout.nslots) def user_setup_slots(self, nslots): assert nslots == 0 diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -57,7 +57,7 @@ # pypy_init_embedded_cffi_module(). if not glob.patched_sys: space.appexec([], """(): - import os + import os, sys sys.stdin = sys.__stdin__ = os.fdopen(0, 'rb', 0) sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0) sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0) diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -5,14 +5,15 @@ class AppTestVMProf(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) - cls.tmpfile = udir.join('test__vmprof.1').open('wb') - cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno()) - cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name) - cls.tmpfile2 = udir.join('test__vmprof.2').open('wb') - cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno()) - cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name) + cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1'))) + cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2'))) def test_import_vmprof(self): + tmpfile = open(self.tmpfilename, 'wb') + tmpfileno = tmpfile.fileno() + tmpfile2 = open(self.tmpfilename2, 'wb') + tmpfileno2 = tmpfile2.fileno() + import struct, sys WORD = struct.calcsize('l') @@ -45,7 +46,7 @@ return count import _vmprof - _vmprof.enable(self.tmpfileno, 0.01) + _vmprof.enable(tmpfileno, 0.01) _vmprof.disable() s = open(self.tmpfilename, 'rb').read() no_of_codes = count(s) @@ -56,7 +57,7 @@ pass """ in d - _vmprof.enable(self.tmpfileno2, 0.01) + _vmprof.enable(tmpfileno2, 0.01) exec """def foo2(): pass diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -34,7 +34,7 @@ import pypy.module.cpyext.pyerrors import pypy.module.cpyext.typeobject import pypy.module.cpyext.object -import pypy.module.cpyext.stringobject +import pypy.module.cpyext.bytesobject import pypy.module.cpyext.tupleobject import pypy.module.cpyext.setobject import pypy.module.cpyext.dictobject @@ -60,7 +60,6 @@ import pypy.module.cpyext.funcobject import pypy.module.cpyext.frameobject import pypy.module.cpyext.classobject -import pypy.module.cpyext.pypyintf import pypy.module.cpyext.memoryobject import pypy.module.cpyext.codecs import pypy.module.cpyext.pyfile diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -9,7 +9,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import ll2ctypes from rpython.rtyper.annlowlevel import llhelper -from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.gensupp import NameManager @@ -30,13 +30,13 @@ from rpython.rlib.rposix import is_valid_fd, validate_fd from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize -from rpython.rlib.exports import export_struct from pypy.module import exceptions from pypy.module.exceptions import interp_exceptions # CPython 2.4 compatibility from py.builtin import BaseException from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rlib import rawrefcount DEBUG_WRAPPER = True @@ -194,7 +194,7 @@ class ApiFunction: def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED, - c_name=None, gil=None): + c_name=None, gil=None, result_borrowed=False): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) @@ -211,17 +211,15 @@ self.argnames = argnames[1:] assert len(self.argnames) == len(self.argtypes) self.gil = gil + self.result_borrowed = result_borrowed + # + def get_llhelper(space): + return llhelper(self.functype, self.get_wrapper(space)) + self.get_llhelper = get_llhelper def _freeze_(self): return True - def get_llhelper(self, space): - llh = getattr(self, '_llhelper', None) - if llh is None: - llh = llhelper(self.functype, self.get_wrapper(space)) - self._llhelper = llh - return llh - @specialize.memo() def get_wrapper(self, space): wrapper = getattr(self, '_wrapper', None) @@ -234,7 +232,7 @@ return wrapper def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', - gil=None): + gil=None, result_borrowed=False): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -263,13 +261,15 @@ rffi.cast(restype, 0) == 0) def decorate(func): + func._always_inline_ = 'try' func_name = func.func_name if header is not None: c_name = None else: c_name = func_name api_function = ApiFunction(argtypes, restype, func, error, - c_name=c_name, gil=gil) + c_name=c_name, gil=gil, + result_borrowed=result_borrowed) func.api_func = api_function if header is not None: @@ -280,6 +280,10 @@ raise ValueError("function %s has no return value for exceptions" % func) def make_unwrapper(catch_exception): + # ZZZ is this whole logic really needed??? It seems to be only + # for RPython code calling PyXxx() functions directly. I would + # think that usually directly calling the function is clean + # enough now names = api_function.argnames types_names_enum_ui = unrolling_iterable(enumerate( zip(api_function.argtypes, @@ -287,56 +291,58 @@ @specialize.ll() def unwrapper(space, *args): - from pypy.module.cpyext.pyobject import Py_DecRef - from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import Reference + from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj + from pypy.module.cpyext.pyobject import from_ref, as_pyobj newargs = () - to_decref = [] + keepalives = () assert len(args) == len(api_function.argtypes) for i, (ARG, is_wrapped) in types_names_enum_ui: input_arg = args[i] if is_PyObject(ARG) and not is_wrapped: - # build a reference - if input_arg is None: - arg = lltype.nullptr(PyObject.TO) - elif isinstance(input_arg, W_Root): - ref = make_ref(space, input_arg) - to_decref.append(ref) - arg = rffi.cast(ARG, ref) + # build a 'PyObject *' (not holding a reference) + if not is_pyobj(input_arg): + keepalives += (input_arg,) + arg = rffi.cast(ARG, as_pyobj(space, input_arg)) + else: + arg = rffi.cast(ARG, input_arg) + elif is_PyObject(ARG) and is_wrapped: + # build a W_Root, possibly from a 'PyObject *' + if is_pyobj(input_arg): + arg = from_ref(space, input_arg) else: arg = input_arg - elif is_PyObject(ARG) and is_wrapped: - # convert to a wrapped object - if input_arg is None: - arg = input_arg - elif isinstance(input_arg, W_Root): - arg = input_arg - else: - try: - arg = from_ref(space, - rffi.cast(PyObject, input_arg)) - except TypeError, e: - err = OperationError(space.w_TypeError, - space.wrap( - "could not cast arg to PyObject")) - if not catch_exception: - raise err - state = space.fromcache(State) - state.set_exception(err) - if is_PyObject(restype): - return None - else: - return api_function.error_value + + ## ZZZ: for is_pyobj: + ## try: + ## arg = from_ref(space, + ## rffi.cast(PyObject, input_arg)) + ## except TypeError, e: + ## err = OperationError(space.w_TypeError, + ## space.wrap( + ## "could not cast arg to PyObject")) + ## if not catch_exception: + ## raise err + ## state = space.fromcache(State) + ## state.set_exception(err) + ## if is_PyObject(restype): + ## return None + ## else: + ## return api_function.error_value else: - # convert to a wrapped object + # arg is not declared as PyObject, no magic arg = input_arg newargs += (arg, ) - try: + if not catch_exception: + try: + res = func(space, *newargs) + finally: + keepalive_until_here(*keepalives) + else: + # non-rpython variant + assert not we_are_translated() try: res = func(space, *newargs) except OperationError, e: - if not catch_exception: - raise if not hasattr(api_function, "error_value"): raise state = space.fromcache(State) @@ -345,21 +351,13 @@ return None else: return api_function.error_value - if not we_are_translated(): - got_integer = isinstance(res, (int, long, float)) - assert got_integer == expect_integer,'got %r not integer' % res - if res is None: - return None - elif isinstance(res, Reference): - return res.get_wrapped(space) - else: - return res - finally: - for arg in to_decref: - Py_DecRef(space, arg) + # 'keepalives' is alive here (it's not rpython) + got_integer = isinstance(res, (int, long, float)) + assert got_integer == expect_integer, ( + 'got %r not integer' % (res,)) + return res unwrapper.func = func unwrapper.api_func = api_function - unwrapper._always_inline_ = 'try' return unwrapper unwrapper_catch = make_unwrapper(True) @@ -501,7 +499,7 @@ GLOBALS['%s#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) for cpyname in '''PyMethodObject PyListObject PyLongObject - PyDictObject PyTupleObject PyClassObject'''.split(): + PyDictObject PyClassObject'''.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s' % (cpyname, )) build_exported_objects() @@ -514,14 +512,16 @@ "PyIntObject*": PyIntObject, "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype] +# Note: as a special case, "PyObject" is the pointer type in RPython, +# corresponding to "PyObject *" in C. We do that only for PyObject. +# For example, "PyTypeObject" is the struct type even in RPython. PyTypeObject = lltype.ForwardReference() PyTypeObjectPtr = lltype.Ptr(PyTypeObject) -# It is important that these PyObjects are allocated in a raw fashion -# Thus we cannot save a forward pointer to the wrapped object -# So we need a forward and backward mapping in our State instance PyObjectStruct = lltype.ForwardReference() PyObject = lltype.Ptr(PyObjectStruct) -PyObjectFields = (("ob_refcnt", lltype.Signed), ("ob_type", PyTypeObjectPtr)) +PyObjectFields = (("ob_refcnt", lltype.Signed), + ("ob_pypy_link", lltype.Signed), + ("ob_type", PyTypeObjectPtr)) PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) cpython_struct('PyObject', PyObjectFields, PyObjectStruct) PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) @@ -618,8 +618,8 @@ @specialize.ll() def wrapper(*args): - from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import Reference + from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj + from pypy.module.cpyext.pyobject import as_pyobj # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer if gil_acquire: @@ -628,6 +628,7 @@ llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py retval = fatal_value boxed_args = () + tb = None try: if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, callable, @@ -635,10 +636,8 @@ for i, (typ, is_wrapped) in argtypes_enum_ui: arg = args[i] if is_PyObject(typ) and is_wrapped: - if arg: - arg_conv = from_ref(space, rffi.cast(PyObject, arg)) - else: - arg_conv = None + assert is_pyobj(arg) + arg_conv = from_ref(space, rffi.cast(PyObject, arg)) else: arg_conv = arg boxed_args += (arg_conv, ) @@ -653,6 +652,7 @@ except BaseException, e: failed = True if not we_are_translated(): + tb = sys.exc_info()[2] message = repr(e) import traceback traceback.print_exc() @@ -671,29 +671,34 @@ retval = error_value elif is_PyObject(callable.api_func.restype): - if result is None: - retval = rffi.cast(callable.api_func.restype, - make_ref(space, None)) - elif isinstance(result, Reference): - retval = result.get_ref(space) - elif not rffi._isllptr(result): - retval = rffi.cast(callable.api_func.restype, - make_ref(space, result)) + if is_pyobj(result): + retval = result else: - retval = result + if result is not None: + if callable.api_func.result_borrowed: + retval = as_pyobj(space, result) + else: + retval = make_ref(space, result) + retval = rffi.cast(callable.api_func.restype, retval) + else: + retval = lltype.nullptr(PyObject.TO) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) except Exception, e: print 'Fatal error in cpyext, CPython compatibility layer, calling', callable.__name__ print 'Either report a bug or consider not using this particular extension' if not we_are_translated(): + if tb is None: + tb = sys.exc_info()[2] import traceback traceback.print_exc() - print str(e) + if sys.stdout == sys.__stdout__: + import pdb; pdb.post_mortem(tb) # we can't do much here, since we're in ctypes, swallow else: print str(e) pypy_debug_catch_fatal_exception() + assert False rffi.stackcounter.stacks_counter -= 1 if gil_release: rgil.release() @@ -827,6 +832,19 @@ outputfilename=str(udir / "module_cache" / "pypyapi")) modulename = py.path.local(eci.libraries[-1]) + def dealloc_trigger(): + from pypy.module.cpyext.pyobject import _Py_Dealloc + print 'dealloc_trigger...' + while True: + ob = rawrefcount.next_dead(PyObject) + if not ob: + break + print ob + _Py_Dealloc(space, ob) + print 'dealloc_trigger DONE' + return "RETRY" + rawrefcount.init(dealloc_trigger) + run_bootstrap_functions(space) # load the bridge, and init structure @@ -836,9 +854,9 @@ space.fromcache(State).install_dll(eci) # populate static data - builder = StaticObjectBuilder(space) + builder = space.fromcache(StaticObjectBuilder) for name, (typ, expr) in GLOBALS.iteritems(): - from pypy.module import cpyext + from pypy.module import cpyext # for the eval() below w_obj = eval(expr) if name.endswith('#'): name = name[:-1] @@ -894,27 +912,44 @@ class StaticObjectBuilder: def __init__(self, space): self.space = space - self.to_attach = [] + self.static_pyobjs = [] + self.static_objs_w = [] + self.cpyext_type_init = None + # + # add a "method" that is overridden in setup_library() + # ('self.static_pyobjs' is completely ignored in that case) + self.get_static_pyobjs = lambda: self.static_pyobjs def prepare(self, py_obj, w_obj): - from pypy.module.cpyext.pyobject import track_reference - py_obj.c_ob_refcnt = 1 - track_reference(self.space, py_obj, w_obj) - self.to_attach.append((py_obj, w_obj)) + "NOT_RPYTHON" + if py_obj: + py_obj.c_ob_refcnt = 1 # 1 for kept immortal + self.static_pyobjs.append(py_obj) + self.static_objs_w.append(w_obj) def attach_all(self): + # this is RPython, called once in pypy-c when it imports cpyext from pypy.module.cpyext.pyobject import get_typedescr, make_ref from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2 + from pypy.module.cpyext.pyobject import track_reference + # space = self.space - space._cpyext_type_init = [] - for py_obj, w_obj in self.to_attach: + static_pyobjs = self.get_static_pyobjs() + static_objs_w = self.static_objs_w + for i in range(len(static_objs_w)): + track_reference(space, static_pyobjs[i], static_objs_w[i]) + # + self.cpyext_type_init = [] + for i in range(len(static_objs_w)): + py_obj = static_pyobjs[i] + w_obj = static_objs_w[i] w_type = space.type(w_obj) - typedescr = get_typedescr(w_type.instancetypedef) + typedescr = get_typedescr(w_type.layout.typedef) py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) typedescr.attach(space, py_obj, w_obj) - cpyext_type_init = space._cpyext_type_init - del space._cpyext_type_init + cpyext_type_init = self.cpyext_type_init + self.cpyext_type_init = None for pto, w_type in cpyext_type_init: finish_type_1(space, pto) finish_type_2(space, pto, w_type) @@ -1067,7 +1102,7 @@ if name.endswith('#'): structs.append('%s %s;' % (typ[:-1], name[:-1])) elif name.startswith('PyExc_'): - structs.append('extern PyTypeObject _%s;' % (name,)) + structs.append('PyTypeObject _%s;' % (name,)) structs.append('PyObject* %s = (PyObject*)&_%s;' % (name, name)) elif typ == 'PyDateTime_CAPI*': structs.append('%s %s = NULL;' % (typ, name)) @@ -1107,7 +1142,7 @@ if not use_micronumpy: return use_micronumpy # import to register api functions by side-effect - import pypy.module.cpyext.ndarrayobject + import pypy.module.cpyext.ndarrayobject global GLOBALS, SYMBOLS_C, separate_module_files GLOBALS["PyArray_Type#"]= ('PyTypeObject*', "space.gettypeobject(W_NDimArray.typedef)") SYMBOLS_C += ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS'] @@ -1116,10 +1151,8 @@ def setup_library(space): "NOT_RPYTHON" - from pypy.module.cpyext.pyobject import make_ref use_micronumpy = setup_micronumpy(space) - - export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) + export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS) from rpython.translator.c.database import LowLevelDatabase db = LowLevelDatabase() @@ -1135,41 +1168,37 @@ run_bootstrap_functions(space) setup_va_functions(eci) - from pypy.module import cpyext # for eval() below - - # Set up the types. Needs a special case, because of the - # immediate cycle involving 'c_ob_type', and because we don't - # want these types to be Py_TPFLAGS_HEAPTYPE. - static_types = {} - for name, (typ, expr) in GLOBALS.items(): - if typ == 'PyTypeObject*': - pto = lltype.malloc(PyTypeObject, immortal=True, - zero=True, flavor='raw') - pto.c_ob_refcnt = 1 - pto.c_tp_basicsize = -1 - static_types[name] = pto - builder = StaticObjectBuilder(space) - for name, pto in static_types.items(): - pto.c_ob_type = static_types['PyType_Type#'] - w_type = eval(GLOBALS[name][1]) - builder.prepare(rffi.cast(PyObject, pto), w_type) - builder.attach_all() - - # populate static data - for name, (typ, expr) in GLOBALS.iteritems(): - name = name.replace("#", "") - if name.startswith('PyExc_'): + # emit uninitialized static data + builder = space.fromcache(StaticObjectBuilder) + lines = ['PyObject *pypy_static_pyobjs[] = {\n'] + include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n'] + for name, (typ, expr) in sorted(GLOBALS.items()): + if name.endswith('#'): + assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*') + typ, name = typ[:-1], name[:-1] + elif name.startswith('PyExc_'): + typ = 'PyTypeObject' name = '_' + name - w_obj = eval(expr) - if typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*'): - struct_ptr = make_ref(space, w_obj) elif typ == 'PyDateTime_CAPI*': continue else: assert False, "Unknown static data: %s %s" % (typ, name) - struct = rffi.cast(get_structtype_for_ctype(typ), struct_ptr)._obj - struct._compilation_info = eci - export_struct(name, struct) + + from pypy.module import cpyext # for the eval() below + w_obj = eval(expr) + builder.prepare(None, w_obj) + lines.append('\t(PyObject *)&%s,\n' % (name,)) + include_lines.append('RPY_EXPORTED %s %s;\n' % (typ, name)) + + lines.append('};\n') + eci2 = CConfig._compilation_info_.merge(ExternalCompilationInfo( + separate_module_sources = [''.join(lines)], + post_include_bits = [''.join(include_lines)], + )) + # override this method to return a pointer to this C array directly + builder.get_static_pyobjs = rffi.CExternVariable( + PyObjectP, 'pypy_static_pyobjs', eci2, c_type='PyObject **', + getter_only=True, declare_as_extern=False) for name, func in FUNCTIONS.iteritems(): newname = mangle_name('PyPy', name) or name @@ -1180,6 +1209,10 @@ trunk_include = pypydir.dirpath() / 'include' copy_header_files(trunk_include, use_micronumpy) +def init_static_data_translated(space): + builder = space.fromcache(StaticObjectBuilder) + builder.attach_all() + def _load_from_cffi(space, name, path, initptr): from pypy.module._cffi_backend import cffi1_module cffi1_module.load_cffi1_module(space, name, path, initptr) @@ -1262,22 +1295,18 @@ @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, True, False)(space, func, *args) - [email protected]() -def generic_cpy_call_dont_decref(space, func, *args): - FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, False, False)(space, func, *args) + return make_generic_cpy_call(FT, False)(space, func, *args) @specialize.ll() def generic_cpy_call_expect_null(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, True, True)(space, func, *args) + return make_generic_cpy_call(FT, True)(space, func, *args) @specialize.memo() -def make_generic_cpy_call(FT, decref_args, expect_null): +def make_generic_cpy_call(FT, expect_null): from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef - from pypy.module.cpyext.pyobject import RefcountState + from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj + from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.module.cpyext.pyerrors import PyErr_Occurred unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) RESULT_TYPE = FT.RESULT @@ -1305,65 +1334,49 @@ @specialize.ll() def generic_cpy_call(space, func, *args): boxed_args = () - to_decref = [] + keepalives = () assert len(args) == len(FT.ARGS) for i, ARG in unrolling_arg_types: arg = args[i] if is_PyObject(ARG): - if arg is None: - boxed_args += (lltype.nullptr(PyObject.TO),) - elif isinstance(arg, W_Root): - ref = make_ref(space, arg) - boxed_args += (ref,) - if decref_args: - to_decref.append(ref) - else: - boxed_args += (arg,) - else: - boxed_args += (arg,) + if not is_pyobj(arg): + keepalives += (arg,) + arg = as_pyobj(space, arg) + boxed_args += (arg,) try: - # create a new container for borrowed references - state = space.fromcache(RefcountState) - old_container = state.swap_borrow_container(None) - try: - # Call the function - result = call_external_function(func, *boxed_args) - finally: - state.swap_borrow_container(old_container) + # Call the function + result = call_external_function(func, *boxed_args) + finally: + keepalive_until_here(*keepalives) - if is_PyObject(RESULT_TYPE): - if result is None: - ret = result - elif isinstance(result, W_Root): - ret = result + if is_PyObject(RESULT_TYPE): + if not is_pyobj(result): + ret = result + else: + # The object reference returned from a C function + # that is called from Python must be an owned reference + # - ownership is transferred from the function to its caller. + if result: + ret = get_w_obj_and_decref(space, result) else: - ret = from_ref(space, result) - # The object reference returned from a C function - # that is called from Python must be an owned reference - # - ownership is transferred from the function to its caller. - if result: - Py_DecRef(space, result) + ret = None - # Check for exception consistency - has_error = PyErr_Occurred(space) is not None - has_result = ret is not None - if has_error and has_result: - raise OperationError(space.w_SystemError, space.wrap( - "An exception was set, but function returned a value")) - elif not expect_null and not has_error and not has_result: - raise OperationError(space.w_SystemError, space.wrap( - "Function returned a NULL result without setting an exception")) + # Check for exception consistency + has_error = PyErr_Occurred(space) is not None + has_result = ret is not None + if has_error and has_result: + raise OperationError(space.w_SystemError, space.wrap( + "An exception was set, but function returned a value")) + elif not expect_null and not has_error and not has_result: + raise OperationError(space.w_SystemError, space.wrap( + "Function returned a NULL result without setting an exception")) - if has_error: - state = space.fromcache(State) - state.check_and_raise_exception() + if has_error: + state = space.fromcache(State) + state.check_and_raise_exception() - return ret - return result - finally: - if decref_args: - for ref in to_decref: - Py_DecRef(space, ref) + return ret + return result + return generic_cpy_call - diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -25,7 +25,7 @@ @bootstrap_function def init_bufferobject(space): "Type description of PyBufferObject" - make_typedescr(space.w_buffer.instancetypedef, + make_typedescr(space.w_buffer.layout.typedef, basestruct=PyBufferObject.TO, attach=buffer_attach, dealloc=buffer_dealloc, diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/bytesobject.py @@ -0,0 +1,319 @@ +from pypy.interpreter.error import OperationError +from rpython.rtyper.lltypesystem import rffi, lltype +from pypy.module.cpyext.api import ( + cpython_api, cpython_struct, bootstrap_function, build_type_checkers, + PyObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) +from pypy.module.cpyext.pyerrors import PyErr_BadArgument +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, + make_typedescr, get_typedescr) + +## +## Implementation of PyStringObject +## ================================ +## +## The problem +## ----------- +## +## PyString_AsString() must return a (non-movable) pointer to the underlying +## buffer, whereas pypy strings are movable. C code may temporarily store +## this address and use it, as long as it owns a reference to the PyObject. +## There is no "release" function to specify that the pointer is not needed +## any more. +## +## Also, the pointer may be used to fill the initial value of string. This is +## valid only when the string was just allocated, and is not used elsewhere. +## +## Solution +## -------- +## +## PyStringObject contains two additional members: the size and a pointer to a +## char buffer; it may be NULL. +## +## - A string allocated by pypy will be converted into a PyStringObject with a +## NULL buffer. The first time PyString_AsString() is called, memory is +## allocated (with flavor='raw') and content is copied. +## +## - A string allocated with PyString_FromStringAndSize(NULL, size) will +## allocate a PyStringObject structure, and a buffer with the specified +## size, but the reference won't be stored in the global map; there is no +## corresponding object in pypy. When from_ref() or Py_INCREF() is called, +## the pypy string is created, and added to the global map of tracked +## objects. The buffer is then supposed to be immutable. +## +## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a +## similar object. +## +## - PyString_Size() doesn't need to force the object. +## +## - There could be an (expensive!) check in from_ref() that the buffer still +## corresponds to the pypy gc-managed string. +## + +PyStringObjectStruct = lltype.ForwardReference() +PyStringObject = lltype.Ptr(PyStringObjectStruct) +PyStringObjectFields = PyObjectFields + \ + (("buffer", rffi.CCHARP), ("size", Py_ssize_t)) +cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) + +@bootstrap_function +def init_stringobject(space): + "Type description of PyStringObject" + make_typedescr(space.w_str.layout.typedef, + basestruct=PyStringObject.TO, + attach=string_attach, + dealloc=string_dealloc, + realize=string_realize) + +PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") + +def new_empty_str(space, length): + """ + Allocate a PyStringObject and its buffer, but without a corresponding + interpreter object. The buffer may be mutated, until string_realize() is + called. Refcount of the result is 1. + """ + typedescr = get_typedescr(space.w_str.layout.typedef) + py_obj = typedescr.allocate(space, space.w_str) + py_str = rffi.cast(PyStringObject, py_obj) + + buflen = length + 1 + py_str.c_size = length + py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, + flavor='raw', zero=True) + return py_str + +def string_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PyStringObject with the given string object. The + buffer must not be modified. + """ + py_str = rffi.cast(PyStringObject, py_obj) + py_str.c_size = len(space.str_w(w_obj)) + py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) + +def string_realize(space, py_obj): + """ + Creates the string in the interpreter. The PyStringObject buffer must not + be modified after this call. + """ + py_str = rffi.cast(PyStringObject, py_obj) + s = rffi.charpsize2str(py_str.c_buffer, py_str.c_size) + w_obj = space.wrap(s) + track_reference(space, py_obj, w_obj) + return w_obj + +@cpython_api([PyObject], lltype.Void, header=None) +def string_dealloc(space, py_obj): + """Frees allocated PyStringObject resources. + """ + py_str = rffi.cast(PyStringObject, py_obj) + if py_str.c_buffer: + lltype.free(py_str.c_buffer, flavor="raw") + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) + +#_______________________________________________________________________ + +@cpython_api([CONST_STRING, Py_ssize_t], PyObject) +def PyString_FromStringAndSize(space, char_p, length): + if char_p: + s = rffi.charpsize2str(char_p, length) + return make_ref(space, space.wrap(s)) + else: + return rffi.cast(PyObject, new_empty_str(space, length)) + +@cpython_api([CONST_STRING], PyObject) +def PyString_FromString(space, char_p): + s = rffi.charp2str(char_p) + return space.wrap(s) + +@cpython_api([PyObject], rffi.CCHARP, error=0) +def PyString_AsString(space, ref): + if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: + pass # typecheck returned "ok" without forcing 'ref' at all + elif not PyString_Check(space, ref): # otherwise, use the alternate way + raise OperationError(space.w_TypeError, space.wrap( + "PyString_AsString only support strings")) + ref_str = rffi.cast(PyStringObject, ref) + if not ref_str.c_buffer: + # copy string buffer + w_str = from_ref(space, ref) + s = space.str_w(w_str) + ref_str.c_buffer = rffi.str2charp(s) + return ref_str.c_buffer + +@cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1) +def PyString_AsStringAndSize(space, ref, buffer, length): + if not PyString_Check(space, ref): + raise OperationError(space.w_TypeError, space.wrap( + "PyString_AsStringAndSize only support strings")) + ref_str = rffi.cast(PyStringObject, ref) + if not ref_str.c_buffer: + # copy string buffer + w_str = from_ref(space, ref) + s = space.str_w(w_str) + ref_str.c_buffer = rffi.str2charp(s) + buffer[0] = ref_str.c_buffer + if length: + length[0] = ref_str.c_size + else: + i = 0 + while ref_str.c_buffer[i] != '\0': + i += 1 + if i != ref_str.c_size: + raise OperationError(space.w_TypeError, space.wrap( + "expected string without null bytes")) + return 0 + +@cpython_api([PyObject], Py_ssize_t, error=-1) +def PyString_Size(space, ref): + if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: + ref = rffi.cast(PyStringObject, ref) + return ref.c_size + else: + w_obj = from_ref(space, ref) + return space.len_w(w_obj) + +@cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) +def _PyString_Resize(space, ref, newsize): + """A way to resize a string object even though it is "immutable". Only use this to + build up a brand new string object; don't use this if the string may already be + known in other parts of the code. It is an error to call this function if the + refcount on the input string object is not one. Pass the address of an existing + string object as an lvalue (it may be written into), and the new size desired. + On success, *string holds the resized string object and 0 is returned; + the address in *string may differ from its input value. If the reallocation + fails, the original string object at *string is deallocated, *string is + set to NULL, a memory exception is set, and -1 is returned. + """ + # XXX always create a new string so far + py_str = rffi.cast(PyStringObject, ref[0]) + if not py_str.c_buffer: + raise OperationError(space.w_SystemError, space.wrap( + "_PyString_Resize called on already created string")) + try: + py_newstr = new_empty_str(space, newsize) + except MemoryError: + Py_DecRef(space, ref[0]) + ref[0] = lltype.nullptr(PyObject.TO) + raise + to_cp = newsize + oldsize = py_str.c_size + if oldsize < newsize: + to_cp = oldsize + for i in range(to_cp): + py_newstr.c_buffer[i] = py_str.c_buffer[i] + Py_DecRef(space, ref[0]) + ref[0] = rffi.cast(PyObject, py_newstr) + return 0 + +@cpython_api([PyObject, PyObject], rffi.INT, error=CANNOT_FAIL) +def _PyString_Eq(space, w_str1, w_str2): + return space.eq_w(w_str1, w_str2) + +@cpython_api([PyObjectP, PyObject], lltype.Void) +def PyString_Concat(space, ref, w_newpart): + """Create a new string object in *string containing the contents of newpart + appended to string; the caller will own the new reference. The reference to + the old value of string will be stolen. If the new string cannot be created, + the old reference to string will still be discarded and the value of + *string will be set to NULL; the appropriate exception will be set.""" + + if not ref[0]: + return + + if w_newpart is None or not PyString_Check(space, ref[0]) or \ + not PyString_Check(space, w_newpart): + Py_DecRef(space, ref[0]) + ref[0] = lltype.nullptr(PyObject.TO) + return + w_str = from_ref(space, ref[0]) + w_newstr = space.add(w_str, w_newpart) + Py_DecRef(space, ref[0]) + ref[0] = make_ref(space, w_newstr) + +@cpython_api([PyObjectP, PyObject], lltype.Void) +def PyString_ConcatAndDel(space, ref, newpart): + """Create a new string object in *string containing the contents of newpart + appended to string. This version decrements the reference count of newpart.""" + PyString_Concat(space, ref, newpart) + Py_DecRef(space, newpart) + +@cpython_api([PyObject, PyObject], PyObject) +def PyString_Format(space, w_format, w_args): + """Return a new string object from format and args. Analogous to format % + args. The args argument must be a tuple.""" + return space.mod(w_format, w_args) + +@cpython_api([CONST_STRING], PyObject) +def PyString_InternFromString(space, string): + """A combination of PyString_FromString() and + PyString_InternInPlace(), returning either a new string object that has + been interned, or a new ("owned") reference to an earlier interned string + object with the same value.""" + s = rffi.charp2str(string) + return space.new_interned_str(s) + +@cpython_api([PyObjectP], lltype.Void) +def PyString_InternInPlace(space, string): + """Intern the argument *string in place. The argument must be the + address of a pointer variable pointing to a Python string object. + If there is an existing interned string that is the same as + *string, it sets *string to it (decrementing the reference count + of the old string object and incrementing the reference count of + the interned string object), otherwise it leaves *string alone and + interns it (incrementing its reference count). (Clarification: + even though there is a lot of talk about reference counts, think + of this function as reference-count-neutral; you own the object + after the call if and only if you owned it before the call.) + + This function is not available in 3.x and does not have a PyBytes + alias.""" + w_str = from_ref(space, string[0]) + w_str = space.new_interned_w_str(w_str) + Py_DecRef(space, string[0]) + string[0] = make_ref(space, w_str) + +@cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) +def PyString_AsEncodedObject(space, w_str, encoding, errors): + """Encode a string object using the codec registered for encoding and return + the result as Python object. encoding and errors have the same meaning as + the parameters of the same name in the string encode() method. The codec to + be used is looked up using the Python codec registry. Return NULL if an + exception was raised by the codec. + + This function is not available in 3.x and does not have a PyBytes alias.""" + if not PyString_Check(space, w_str): + PyErr_BadArgument(space) + + w_encoding = w_errors = None + if encoding: + w_encoding = space.wrap(rffi.charp2str(encoding)) + if errors: + w_errors = space.wrap(rffi.charp2str(errors)) + return space.call_method(w_str, 'encode', w_encoding, w_errors) + +@cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) +def PyString_AsDecodedObject(space, w_str, encoding, errors): + """Decode a string object by passing it to the codec registered + for encoding and return the result as Python object. encoding and + errors have the same meaning as the parameters of the same name in + the string encode() method. The codec to be used is looked up + using the Python codec registry. Return NULL if an exception was + raised by the codec. + + This function is not available in 3.x and does not have a PyBytes alias.""" + if not PyString_Check(space, w_str): + PyErr_BadArgument(space) + + w_encoding = w_errors = None + if encoding: + w_encoding = space.wrap(rffi.charp2str(encoding)) + if errors: + w_errors = space.wrap(rffi.charp2str(errors)) + return space.call_method(w_str, "decode", w_encoding, w_errors) + +@cpython_api([PyObject, PyObject], PyObject) +def _PyString_Join(space, w_sep, w_seq): + return space.call_method(w_sep, 'join', w_seq) diff --git a/pypy/module/cpyext/complexobject.py b/pypy/module/cpyext/complexobject.py --- a/pypy/module/cpyext/complexobject.py +++ b/pypy/module/cpyext/complexobject.py @@ -43,7 +43,7 @@ # lltype does not handle functions returning a structure. This implements a # helper function, which takes as argument a reference to the return value. -@cpython_api([PyObject, Py_complex_ptr], lltype.Void) +@cpython_api([PyObject, Py_complex_ptr], rffi.INT_real, error=-1) def _PyComplex_AsCComplex(space, w_obj, result): """Return the Py_complex value of the complex number op. @@ -60,7 +60,7 @@ # if the above did not work, interpret obj as a float giving the # real part of the result, and fill in the imaginary part as 0. result.c_real = PyFloat_AsDouble(space, w_obj) # -1 on failure - return + return 0 if not PyComplex_Check(space, w_obj): raise OperationError(space.w_TypeError, space.wrap( @@ -69,3 +69,4 @@ assert isinstance(w_obj, W_ComplexObject) result.c_real = w_obj.realval result.c_imag = w_obj.imagval + return 0 diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py --- a/pypy/module/cpyext/dictobject.py +++ b/pypy/module/cpyext/dictobject.py @@ -2,8 +2,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t, Py_ssize_tP, CONST_STRING) -from pypy.module.cpyext.pyobject import PyObject, PyObjectP, borrow_from -from pypy.module.cpyext.pyobject import RefcountState +from pypy.module.cpyext.pyobject import PyObject, PyObjectP, as_pyobj from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import specialize @@ -14,13 +13,17 @@ PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict") -@cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) +@cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL, + result_borrowed=True) def PyDict_GetItem(space, w_dict, w_key): try: w_res = space.getitem(w_dict, w_key) except: return None - return borrow_from(w_dict, w_res) + # NOTE: this works so far because all our dict strategies store + # *values* as full objects, which stay alive as long as the dict is + # alive and not modified. So we can return a borrowed ref. + return w_res @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_SetItem(space, w_dict, w_key, w_obj): @@ -47,7 +50,8 @@ else: PyErr_BadInternalCall(space) -@cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL) +@cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL, + result_borrowed=True) def PyDict_GetItemString(space, w_dict, key): """This is the same as PyDict_GetItem(), but key is specified as a char*, rather than a PyObject*.""" @@ -55,9 +59,10 @@ w_res = space.finditem_str(w_dict, rffi.charp2str(key)) except: w_res = None - if w_res is None: - return None - return borrow_from(w_dict, w_res) + # NOTE: this works so far because all our dict strategies store + # *values* as full objects, which stay alive as long as the dict is + # alive and not modified. So we can return a borrowed ref. + return w_res @cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) def PyDict_DelItemString(space, w_dict, key_ptr): @@ -170,10 +175,13 @@ if w_dict is None: return 0 - # Note: this is not efficient. Storing an iterator would probably + # XXX XXX PyDict_Next is not efficient. Storing an iterator would probably # work, but we can't work out how to not leak it if iteration does - # not complete. + # not complete. Alternatively, we could add some RPython-only + # dict-iterator method to move forward by N steps. + w_dict.ensure_object_strategy() # make sure both keys and values can + # be borrwed try: w_iter = space.call_method(space.w_dict, "iteritems", w_dict) pos = ppos[0] @@ -183,11 +191,10 @@ w_item = space.call_method(w_iter, "next") w_key, w_value = space.fixedview(w_item, 2) - state = space.fromcache(RefcountState) if pkey: - pkey[0] = state.make_borrowed(w_dict, w_key) + pkey[0] = as_pyobj(space, w_key) if pvalue: - pvalue[0] = state.make_borrowed(w_dict, w_value) + pvalue[0] = as_pyobj(space, w_value) ppos[0] += 1 except OperationError, e: if not e.match(space, space.w_StopIteration): diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -4,7 +4,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, cpython_struct, is_valid_fp) -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject from pypy.module.__builtin__ import compiling @@ -23,7 +23,7 @@ def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): return space.call(w_obj, w_arg, w_kwds) -@cpython_api([], PyObject) +@cpython_api([], PyObject, result_borrowed=True) def PyEval_GetBuiltins(space): """Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is @@ -36,25 +36,25 @@ w_builtins = w_builtins.getdict(space) else: w_builtins = space.builtin.getdict(space) - return borrow_from(None, w_builtins) + return w_builtins # borrowed ref in all cases -@cpython_api([], PyObject, error=CANNOT_FAIL) +@cpython_api([], PyObject, error=CANNOT_FAIL, result_borrowed=True) def PyEval_GetLocals(space): """Return a dictionary of the local variables in the current execution frame, or NULL if no frame is currently executing.""" caller = space.getexecutioncontext().gettopframe_nohidden() if caller is None: return None - return borrow_from(None, caller.getdictscope()) + return caller.getdictscope() # borrowed ref -@cpython_api([], PyObject, error=CANNOT_FAIL) +@cpython_api([], PyObject, error=CANNOT_FAIL, result_borrowed=True) def PyEval_GetGlobals(space): """Return a dictionary of the global variables in the current execution frame, or NULL if no frame is currently executing.""" caller = space.getexecutioncontext().gettopframe_nohidden() if caller is None: return None _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
