Author: Richard Plangger <planri...@gmail.com> Branch: new-jit-log Changeset: r85302:bcdc23248f13 Date: 2016-06-21 12:57 +0200 http://bitbucket.org/pypy/pypy/changeset/bcdc23248f13/
Log: catchup default diff too long, truncating to 2000 out of 3966 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -26,3 +26,4 @@ 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 +7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1 diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.9 +Version: 0.4.10 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.9" +__version__ = "0.4.10" # ____________________________________________________________ # Exceptions diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -76,6 +76,20 @@ def pytest_pycollect_makemodule(path, parent): return PyPyModule(path, parent) +def is_applevel(item): + from pypy.tool.pytest.apptest import AppTestFunction + return isinstance(item, AppTestFunction) + +def pytest_collection_modifyitems(config, items): + if config.option.runappdirect: + return + for item in items: + if isinstance(item, py.test.Function): + if is_applevel(item): + item.add_marker('applevel') + else: + item.add_marker('interplevel') + class PyPyModule(py.test.collect.Module): """ we take care of collecting classes both at app level and at interp-level (because we need to stick a space @@ -110,9 +124,6 @@ if name.startswith('AppTest'): from pypy.tool.pytest.apptest import AppClassCollector return AppClassCollector(name, parent=self) - else: - from pypy.tool.pytest.inttest import IntClassCollector - return IntClassCollector(name, parent=self) elif hasattr(obj, 'func_code') and self.funcnamefilter(name): if name.startswith('app_test_'): @@ -120,11 +131,7 @@ "generator app level functions? you must be joking" from pypy.tool.pytest.apptest import AppTestFunction return AppTestFunction(name, parent=self) - elif obj.func_code.co_flags & 32: # generator function - return pytest.Generator(name, parent=self) - else: - from pypy.tool.pytest.inttest import IntTestFunction - return IntTestFunction(name, parent=self) + return super(PyPyModule, self).makeitem(name, obj) def skip_on_missing_buildoption(**ropts): __tracebackhide__ = True @@ -153,28 +160,19 @@ def pytest_runtest_setup(__multicall__, item): if isinstance(item, py.test.collect.Function): - appclass = item.getparent(PyPyClassCollector) + appclass = item.getparent(py.test.Class) if appclass is not None: # Make cls.space and cls.runappdirect available in tests. spaceconfig = getattr(appclass.obj, 'spaceconfig', None) if spaceconfig is not None: from pypy.tool.pytest.objspace import gettestobjspace appclass.obj.space = gettestobjspace(**spaceconfig) + else: + appclass.obj.space = LazyObjSpaceGetter() appclass.obj.runappdirect = option.runappdirect __multicall__.execute() -class PyPyClassCollector(py.test.collect.Class): - # All pypy Test classes have a "space" member. - def setup(self): - cls = self.obj - if not hasattr(cls, 'spaceconfig'): - cls.space = LazyObjSpaceGetter() - else: - assert hasattr(cls, 'space') # set by pytest_runtest_setup - super(PyPyClassCollector, self).setup() - - def pytest_ignore_collect(path): return path.check(link=1) diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -315,13 +315,28 @@ - ``complex`` + - ``str`` (empty or single-character strings only) + + - ``unicode`` (empty or single-character strings only) + + - ``tuple`` (empty tuples only) + + - ``frozenset`` (empty frozenset only) + This change requires some changes to ``id`` as well. ``id`` fulfills the following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the above types will return a value that is computed from the argument, and can thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long). -Notably missing from the list above are ``str`` and ``unicode``. If your -code relies on comparing strings with ``is``, then it might break in PyPy. +Note that strings of length 2 or greater can be equal without being +identical. Similarly, ``x is (2,)`` is not necessarily true even if +``x`` contains a tuple and ``x == (2,)``. The uniqueness rules apply +only to the particular cases described above. The ``str``, ``unicode``, +``tuple`` and ``frozenset`` rules were added in PyPy 5.4; before that, a +test like ``if x is "?"`` or ``if x is ()`` could fail even if ``x`` was +equal to ``"?"`` or ``()``. The new behavior added in PyPy 5.4 is +closer to CPython's, which caches precisely the empty tuple/frozenset, +and (generally but not always) the strings and unicodes of length <= 1. Note that for floats there "``is``" only one object per "bit pattern" of the float. So ``float('nan') is float('nan')`` is true on PyPy, diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.3.1.rst release-pypy2.7-v5.3.0.rst release-5.1.1.rst release-5.1.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.3.1.rst whatsnew-pypy2-5.3.0.rst whatsnew-5.1.0.rst whatsnew-5.0.0.rst diff --git a/pypy/doc/release-pypy2.7-v5.3.1.rst b/pypy/doc/release-pypy2.7-v5.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.3.1.rst @@ -0,0 +1,41 @@ +========== +PyPy 5.3.1 +========== + +We have released a bugfix for PyPy2.7-v5.3.0, released last week, +due to issues_ reported by users. + +Thanks to those who reported the issues. + +.. _issues: http://doc.pypy.org/en/latest/whatsnew-pypy2-5.3.1.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,10 +1,17 @@ -========================= +========================== What's new in PyPy2.7 5.3+ -========================= +========================== .. this is a revision shortly after release-pypy2.7-v5.3 .. startrev: 873218a739f1 +.. 418b05f95db5 +Improve CPython compatibility for ``is``. Now code like ``if x is ():`` +works the same way as it does on CPython. See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id . + +.. pull request #455 +Add sys.{get,set}dlopenflags, for cpyext extensions. + .. branch: fix-gen-dfa Resolves an issue with the generator script to build the dfa for Python syntax. @@ -19,3 +26,19 @@ .. branch: s390x-5.3-catchup Implement the backend related changes for s390x. + +.. branch: incminimark-ll_assert +.. branch: vmprof-openbsd + +.. branch: testing-cleanup + +Simplify handling of interp-level tests and make it more forward- +compatible. + +.. branch: pyfile-tell +Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile + +.. branch: rw-PyString_AS_STRING +Allow rw access to the char* returned from PyString_AS_STRING, also refactor +PyStringObject to look like cpython's and allow subclassing PyString_Type and +PyUnicode_Type diff --git a/pypy/doc/whatsnew-pypy2-5.3.1.rst b/pypy/doc/whatsnew-pypy2-5.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.3.1.rst @@ -0,0 +1,15 @@ +=========================== +What's new in PyPy2.7 5.3.1 +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.3.0 +.. startrev: f4d726d1a010 + + +A bug-fix release, merging these changes: + + * Add include guards to pymem.h, fixes issue #2321 + + * Make vmprof build on OpenBSD, from pull request #456 + + * Fix ``bytearray('').replace('a', 'ab')``, issue #2324 diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -458,14 +458,17 @@ decl = str(decl) + "\n" yield self.st, decl, 'x', (1, 2, 3, 4) + def test_closure_error(self): source = """if 1: def f(a): del a def x(): a """ - exc = py.test.raises(SyntaxError, self.run, source).value - assert exc.msg == "Can't delete variable used in nested scopes: 'a'" + with py.test.raises(SyntaxError) as excinfo: + self.run(source) + msg = excinfo.value.msg + assert msg == "Can't delete variable used in nested scopes: 'a'" def test_try_except_finally(self): yield self.simple_test, """ @@ -879,7 +882,20 @@ """ self.simple_test(source, 'ok', 1) - def test_remove_docstring(self): + @py.test.mark.parametrize('expr, result', [ + ("f1.__doc__", None), + ("f2.__doc__", 'docstring'), + ("f2()", 'docstring'), + ("f3.__doc__", None), + ("f3()", 'bar'), + ("C1.__doc__", None), + ("C2.__doc__", 'docstring'), + ("C3.field", 'not docstring'), + ("C4.field", 'docstring'), + ("C4.__doc__", 'docstring'), + ("C4.__doc__", 'docstring'), + ("__doc__", None),]) + def test_remove_docstring(self, expr, result): source = '"module_docstring"\n' + """if 1: def f1(): 'docstring' @@ -903,19 +919,7 @@ code_w.remove_docstrings(self.space) dict_w = self.space.newdict(); code_w.exec_code(self.space, dict_w, dict_w) - - yield self.check, dict_w, "f1.__doc__", None - yield self.check, dict_w, "f2.__doc__", 'docstring' - yield self.check, dict_w, "f2()", 'docstring' - yield self.check, dict_w, "f3.__doc__", None - yield self.check, dict_w, "f3()", 'bar' - yield self.check, dict_w, "C1.__doc__", None - yield self.check, dict_w, "C2.__doc__", 'docstring' - yield self.check, dict_w, "C3.field", 'not docstring' - yield self.check, dict_w, "C4.field", 'docstring' - yield self.check, dict_w, "C4.__doc__", 'docstring' - yield self.check, dict_w, "C4.__doc__", 'docstring' - yield self.check, dict_w, "__doc__", None + self.check(dict_w, expr, result) def test_assert_skipping(self): space = self.space @@ -1111,7 +1115,7 @@ return d['f'](5) """) assert 'generator' in space.str_w(space.repr(w_generator)) - + def test_list_comprehension(self): source = "def f(): [i for i in l]" source2 = "def f(): [i for i in l for j in l]" diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -671,13 +671,11 @@ class AppTestSocketTCP: HOST = 'localhost' - - def setup_class(cls): - cls.space = space + spaceconfig = {'usemodules': ['_socket', 'array']} def setup_method(self, method): - w_HOST = space.wrap(self.HOST) - self.w_serv = space.appexec([w_socket, w_HOST], + w_HOST = self.space.wrap(self.HOST) + self.w_serv =self.space.appexec([w_socket, w_HOST], '''(_socket, HOST): serv = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) serv.bind((HOST, 0)) @@ -687,7 +685,7 @@ def teardown_method(self, method): if hasattr(self, 'w_serv'): - space.appexec([self.w_serv], '(serv): serv.close()') + self.space.appexec([self.w_serv], '(serv): serv.close()') self.w_serv = None def test_timeout(self): @@ -803,8 +801,7 @@ class AppTestErrno: - def setup_class(cls): - cls.space = space + spaceconfig = {'usemodules': ['_socket']} def test_errno(self): from socket import socket, AF_INET, SOCK_STREAM, error diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -3,8 +3,9 @@ from pypy.tool.pytest.objspace import gettestobjspace class AppTestVMProf(object): + spaceconfig = {'usemodules': ['_vmprof', 'struct']} + def setup_class(cls): - cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1'))) cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2'))) @@ -17,7 +18,7 @@ import struct, sys, gc WORD = struct.calcsize('l') - + def count(s): i = 0 count = 0 @@ -44,7 +45,7 @@ else: raise AssertionError(ord(s[i])) return count - + import _vmprof gc.collect() # try to make the weakref list deterministic gc.collect() # by freeing all dead code objects diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -800,6 +800,21 @@ pypy_debug_catch_fatal_exception() assert False +def _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid): + from rpython.rlib import rgil + # see "Handling of the GIL" above + assert cpyext_glob_tid_ptr[0] == 0 + if pygilstate_release: + from pypy.module.cpyext import pystate + unlock = (gilstate == pystate.PyGILState_UNLOCKED) + else: + unlock = gil_release or _gil_auto + if unlock: + rgil.release() + else: + cpyext_glob_tid_ptr[0] = tid + + def make_wrapper_second_level(space, argtypesw, restype, result_kind, error_value, gil): from rpython.rlib import rgil @@ -827,6 +842,7 @@ def wrapper_second_level(callable, pname, *args): from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj from pypy.module.cpyext.pyobject import as_pyobj + from pypy.module.cpyext import pystate # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer @@ -839,7 +855,6 @@ rgil.acquire() assert cpyext_glob_tid_ptr[0] == 0 elif pygilstate_ensure: - from pypy.module.cpyext import pystate if cpyext_glob_tid_ptr[0] == tid: cpyext_glob_tid_ptr[0] = 0 args += (pystate.PyGILState_LOCKED,) @@ -850,6 +865,10 @@ if cpyext_glob_tid_ptr[0] != tid: no_gil_error(pname) cpyext_glob_tid_ptr[0] = 0 + if pygilstate_release: + gilstate = rffi.cast(lltype.Signed, args[-1]) + else: + gilstate = pystate.PyGILState_IGNORE rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -919,24 +938,13 @@ except Exception as e: unexpected_exception(pname, e, tb) + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return fatal_value assert lltype.typeOf(retval) == restype rffi.stackcounter.stacks_counter -= 1 - # see "Handling of the GIL" above - assert cpyext_glob_tid_ptr[0] == 0 - if pygilstate_release: - from pypy.module.cpyext import pystate - arg = rffi.cast(lltype.Signed, args[-1]) - unlock = (arg == pystate.PyGILState_UNLOCKED) - else: - unlock = gil_release or _gil_auto - if unlock: - rgil.release() - else: - cpyext_glob_tid_ptr[0] = tid - + _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid) return retval wrapper_second_level._dont_inline_ = True @@ -1202,8 +1210,6 @@ cpyext_type_init = self.cpyext_type_init self.cpyext_type_init = None for pto, w_type in cpyext_type_init: - if space.is_w(w_type, space.w_str): - pto.c_tp_itemsize = 1 finish_type_1(space, pto) finish_type_2(space, pto, w_type) @@ -1512,7 +1518,7 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname) + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -6,7 +6,9 @@ from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, - make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref) + make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref, + pyobj_has_w_obj) +from pypy.objspace.std.bytesobject import W_BytesObject ## ## Implementation of PyStringObject @@ -16,7 +18,7 @@ ## ----------- ## ## PyString_AsString() must return a (non-movable) pointer to the underlying -## buffer, whereas pypy strings are movable. C code may temporarily store +## ob_sval, whereas pypy strings are movable. C code may temporarily store ## this address and use it, as long as it owns a reference to the PyObject. ## There is no "release" function to specify that the pointer is not needed ## any more. @@ -28,7 +30,7 @@ ## -------- ## ## PyStringObject contains two additional members: the ob_size and a pointer to a -## char buffer; it may be NULL. +## char ob_sval; it may be NULL. ## ## - A string allocated by pypy will be converted into a PyStringObject with a ## NULL buffer. The first time PyString_AsString() is called, memory is @@ -41,6 +43,9 @@ ## the pypy string is created, and added to the global map of tracked ## objects. The buffer is then supposed to be immutable. ## +##- A buffer obtained from PyString_AS_STRING() could be mutable iff +## there is no corresponding pypy object for the string +## ## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a ## similar object. ## @@ -53,7 +58,7 @@ PyStringObjectStruct = lltype.ForwardReference() PyStringObject = lltype.Ptr(PyStringObjectStruct) PyStringObjectFields = PyVarObjectFields + \ - (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("buffer", rffi.CCHARP)) + (("ob_shash", rffi.LONG), ("ob_sstate", rffi.INT), ("ob_sval", rffi.CArray(lltype.Char))) cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) @bootstrap_function @@ -69,44 +74,43 @@ def new_empty_str(space, length): """ - Allocate a PyStringObject and its buffer, but without a corresponding - interpreter object. The buffer may be mutated, until string_realize() is + Allocate a PyStringObject and its ob_sval, but without a corresponding + interpreter object. The ob_sval may be mutated, until string_realize() is called. Refcount of the result is 1. """ typedescr = get_typedescr(space.w_str.layout.typedef) - py_obj = typedescr.allocate(space, space.w_str) + py_obj = typedescr.allocate(space, space.w_str, length) py_str = rffi.cast(PyStringObject, py_obj) - - buflen = length + 1 - py_str.c_ob_size = length - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, - flavor='raw', zero=True, - add_memory_pressure=True) + py_str.c_ob_shash = -1 py_str.c_ob_sstate = rffi.cast(rffi.INT, 0) # SSTATE_NOT_INTERNED return py_str def string_attach(space, py_obj, w_obj): """ - Fills a newly allocated PyStringObject with the given string object. The - buffer must not be modified. + Copy RPython string object contents to a PyStringObject. The + c_ob_sval must not be modified. """ py_str = rffi.cast(PyStringObject, py_obj) - py_str.c_ob_size = len(space.str_w(w_obj)) - py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) + s = space.str_w(w_obj) + if py_str.c_ob_size < len(s): + raise oefmt(space.w_ValueError, + "string_attach called on object with ob_size %d but trying to store %d", + py_str.c_ob_size, len(s)) + rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s)) + py_str.c_ob_sval[len(s)] = '\0' py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL def string_realize(space, py_obj): """ - Creates the string in the interpreter. The PyStringObject buffer must not + Creates the string in the interpreter. The PyStringObject ob_sval must not be modified after this call. """ py_str = rffi.cast(PyStringObject, py_obj) - if not py_str.c_buffer: - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, py_str.c_ob_size + 1, - flavor='raw', zero=True) - s = rffi.charpsize2str(py_str.c_buffer, py_str.c_ob_size) - w_obj = space.wrap(s) + s = rffi.charpsize2str(py_str.c_ob_sval, py_str.c_ob_size) + w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) + w_obj = space.allocate_instance(W_BytesObject, w_type) + w_obj.__init__(s) py_str.c_ob_shash = space.hash_w(w_obj) py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL track_reference(space, py_obj, w_obj) @@ -116,9 +120,6 @@ def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ - py_str = rffi.cast(PyStringObject, py_obj) - if py_str.c_buffer: - lltype.free(py_str.c_buffer, flavor="raw") from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) @@ -139,6 +140,9 @@ @cpython_api([PyObject], rffi.CCHARP, error=0) def PyString_AsString(space, ref): + return _PyString_AsString(space, ref) + +def _PyString_AsString(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: pass # typecheck returned "ok" without forcing 'ref' at all elif not PyString_Check(space, ref): # otherwise, use the alternate way @@ -151,15 +155,23 @@ "expected string or Unicode object, %T found", from_ref(space, ref)) ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - return ref_str.c_buffer + if not pyobj_has_w_obj(ref): + # XXX Force the ref? + string_realize(space, ref) + return ref_str.c_ob_sval + +@cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) +def PyString_AS_STRING(space, void_ref): + ref = rffi.cast(PyObject, void_ref) + # if no w_str is associated with this ref, + # return the c-level ptr as RW + if not pyobj_has_w_obj(ref): + py_str = rffi.cast(PyStringObject, ref) + return py_str.c_ob_sval + return _PyString_AsString(space, ref) @cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1) -def PyString_AsStringAndSize(space, ref, buffer, length): +def PyString_AsStringAndSize(space, ref, data, length): if not PyString_Check(space, ref): from pypy.module.cpyext.unicodeobject import ( PyUnicode_Check, _PyUnicode_AsDefaultEncodedString) @@ -169,18 +181,16 @@ raise oefmt(space.w_TypeError, "expected string or Unicode object, %T found", from_ref(space, ref)) + if not pyobj_has_w_obj(ref): + # force the ref + string_realize(space, ref) ref_str = rffi.cast(PyStringObject, ref) - if not ref_str.c_buffer: - # copy string buffer - w_str = from_ref(space, ref) - s = space.str_w(w_str) - ref_str.c_buffer = rffi.str2charp(s) - buffer[0] = ref_str.c_buffer + data[0] = ref_str.c_ob_sval if length: length[0] = ref_str.c_ob_size else: i = 0 - while ref_str.c_buffer[i] != '\0': + while ref_str.c_ob_sval[i] != '\0': i += 1 if i != ref_str.c_ob_size: raise oefmt(space.w_TypeError, @@ -209,10 +219,10 @@ set to NULL, a memory exception is set, and -1 is returned. """ # XXX always create a new string so far - py_str = rffi.cast(PyStringObject, ref[0]) - if not py_str.c_buffer: + if pyobj_has_w_obj(ref[0]): raise oefmt(space.w_SystemError, "_PyString_Resize called on already created string") + py_str = rffi.cast(PyStringObject, ref[0]) try: py_newstr = new_empty_str(space, newsize) except MemoryError: @@ -224,7 +234,7 @@ if oldsize < newsize: to_cp = oldsize for i in range(to_cp): - py_newstr.c_buffer[i] = py_str.c_buffer[i] + py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i] Py_DecRef(space, ref[0]) ref[0] = rffi.cast(PyObject, py_newstr) return 0 diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.3.1-alpha0" -#define PYPY_VERSION_NUM 0x05030100 +#define PYPY_VERSION "5.3.2-alpha0" +#define PYPY_VERSION_NUM 0x05030200 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h --- a/pypy/module/cpyext/include/pymem.h +++ b/pypy/module/cpyext/include/pymem.h @@ -1,5 +1,11 @@ #include <stdlib.h> +#ifndef Py_PYMEM_H +#define Py_PYMEM_H + +#ifdef __cplusplus +extern "C" { +#endif #define PyMem_MALLOC(n) malloc((n) ? (n) : 1) #define PyMem_REALLOC(p, n) realloc((p), (n) ? (n) : 1) @@ -44,3 +50,9 @@ */ #define PyMem_Del PyMem_Free #define PyMem_DEL PyMem_FREE + +#ifdef __cplusplus +} +#endif + +#endif /* !Py_PYMEM_H */ diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h --- a/pypy/module/cpyext/include/stringobject.h +++ b/pypy/module/cpyext/include/stringobject.h @@ -10,7 +10,6 @@ #include <stdarg.h> #define PyString_GET_SIZE(op) PyString_Size((PyObject*)(op)) -#define PyString_AS_STRING(op) PyString_AsString((PyObject*)(op)) /* Type PyStringObject represents a character string. An extra zero byte is reserved at the end to ensure it is zero-terminated, but a size is @@ -41,12 +40,11 @@ PyObject_VAR_HEAD long ob_shash; int ob_sstate; - char * buffer; /* change the name from cpython so all non-api c access is thwarted */ + char ob_sval[1]; /* Invariants - * (not relevant in PyPy, all stringobjects are backed by a pypy object) - * buffer contains space for 'ob_size+1' elements. - * buffer[ob_size] == 0. + * ob_sval contains space for 'ob_size+1' elements. + * ob_sval[ob_size] == 0. * ob_shash is the hash of the string or -1 if not computed yet. * ob_sstate != 0 iff the string object is in stringobject.c's * 'interned' dictionary; in this case the two references diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -55,6 +55,7 @@ if not PyFile_Check(space, w_p): raise oefmt(space.w_IOError, 'first argument must be an open file') assert isinstance(w_p, W_File) + w_p.stream.flush_buffers() try: fd = space.int_w(space.call_method(w_p, 'fileno')) mode = w_p.mode diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -7,7 +7,7 @@ from pypy.module.cpyext.api import ( cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR, CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject, - INTERPLEVEL_API) + INTERPLEVEL_API, PyVarObject) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject @@ -47,13 +47,16 @@ size = pytype.c_tp_basicsize else: size = rffi.sizeof(self.basestruct) - if itemcount and w_type is not space.w_str: + if pytype.c_tp_itemsize: size += itemcount * pytype.c_tp_itemsize assert size >= rffi.sizeof(PyObject.TO) buf = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True, add_memory_pressure=True) pyobj = rffi.cast(PyObject, buf) + if pytype.c_tp_itemsize: + pyvarobj = rffi.cast(PyVarObject, pyobj) + pyvarobj.c_ob_size = itemcount pyobj.c_ob_refcnt = 1 #pyobj.c_ob_pypy_link should get assigned very quickly pyobj.c_ob_type = pytype @@ -152,13 +155,18 @@ class InvalidPointerException(Exception): pass -def create_ref(space, w_obj, itemcount=0): +def create_ref(space, w_obj): """ Allocates a PyObject, and fills its fields with info from the given interpreter object. """ w_type = space.type(w_obj) + pytype = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type)) typedescr = get_typedescr(w_obj.typedef) + if pytype.c_tp_itemsize != 0: + itemcount = space.len_w(w_obj) # PyStringObject and subclasses + else: + itemcount = 0 py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) track_reference(space, py_obj, w_obj) # diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -207,6 +207,7 @@ PyGILState_STATE = rffi.INT PyGILState_LOCKED = 0 PyGILState_UNLOCKED = 1 +PyGILState_IGNORE = 2 ExecutionContext.cpyext_gilstate_counter_noleave = 0 diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c --- a/pypy/module/cpyext/src/stringobject.c +++ b/pypy/module/cpyext/src/stringobject.c @@ -107,7 +107,7 @@ if (!string) return NULL; - s = PyString_AsString(string); + s = PyString_AS_STRING(string); for (f = format; *f; f++) { if (*f == '%') { diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,4 +1,4 @@ -import py +import os import pytest def pytest_configure(config): @@ -21,3 +21,14 @@ def pytest_funcarg__api(request): return request.cls.api +if os.name == 'nt': + @pytest.yield_fixture(autouse=True, scope='session') + def prevent_dialog_box(): + """Do not open dreaded dialog box on segfault on Windows""" + import ctypes + SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN + old_err_mode = ctypes.windll.kernel32.GetErrorMode() + new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX + ctypes.windll.kernel32.SetErrorMode(new_err_mode) + yield + ctypes.windll.kernel32.SetErrorMode(old_err_mode) diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -43,15 +43,6 @@ /* foo methods */ -static void -foo_dealloc(fooobject *foop) -{ - PyObject_Del(foop); -} - - -/* foo methods-as-attributes */ - static PyObject * foo_copy(fooobject *self) { @@ -195,7 +186,7 @@ sizeof(fooobject), /*tp_size*/ 0, /*tp_itemsize*/ /* methods */ - (destructor)foo_dealloc, /*tp_dealloc*/ + 0, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -463,9 +454,9 @@ PyTypeObject InitErrType = { PyObject_HEAD_INIT(NULL) 0, - "foo.InitErr", - sizeof(PyObject), - 0, + "foo.InitErrType", + sizeof(PyObject),/*tp_basicsize*/ + 0, /*tp_itemsize*/ 0, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ @@ -508,12 +499,12 @@ 0, /*tp_dictoffset*/ initerrtype_init, /*tp_init*/ - 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_alloc*/ 0, /*tp_new*/ - 0, /*tp_free Low-level free-memory routine */ - 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ 0, /*tp_bases*/ - 0, /*tp_mro method resolution order */ + 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0 /*tp_weaklist*/ diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/support.py @@ -0,0 +1,68 @@ +import os +import py +from sys import platform + +if os.name != 'nt': + so_ext = 'so' +else: + so_ext = 'dll' + +def c_compile(cfilenames, outputfilename, + compile_extra=None, link_extra=None, + include_dirs=None, libraries=None, library_dirs=None): + compile_extra = compile_extra or [] + link_extra = link_extra or [] + include_dirs = include_dirs or [] + libraries = libraries or [] + library_dirs = library_dirs or [] + if platform == 'win32': + link_extra = link_extra + ['/DEBUG'] # generate .pdb file + if platform == 'darwin': + # support Fink & Darwinports + for s in ('/sw/', '/opt/local/'): + if (s + 'include' not in include_dirs + and os.path.exists(s + 'include')): + include_dirs.append(s + 'include') + if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'): + library_dirs.append(s + 'lib') + + outputfilename = py.path.local(outputfilename).new(ext=so_ext) + saved_environ = os.environ.copy() + try: + _build( + cfilenames, outputfilename, + compile_extra, link_extra, + include_dirs, libraries, library_dirs) + finally: + # workaround for a distutils bugs where some env vars can + # become longer and longer every time it is used + for key, value in saved_environ.items(): + if os.environ.get(key) != value: + os.environ[key] = value + return outputfilename + +def _build(cfilenames, outputfilename, compile_extra, link_extra, + include_dirs, libraries, library_dirs): + from distutils.ccompiler import new_compiler + from distutils import sysconfig + compiler = new_compiler(force=1) + sysconfig.customize_compiler(compiler) # XXX + objects = [] + for cfile in cfilenames: + cfile = py.path.local(cfile) + old = cfile.dirpath().chdir() + try: + res = compiler.compile([cfile.basename], + include_dirs=include_dirs, extra_preargs=compile_extra) + assert len(res) == 1 + cobjfile = py.path.local(res[0]) + assert cobjfile.check() + objects.append(str(cobjfile)) + finally: + old.chdir() + + compiler.link_shared_object( + objects, str(outputfilename), + libraries=libraries, + extra_preargs=link_extra, + library_dirs=library_dirs) diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -1,5 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + class AppTestStringObject(AppTestCpythonExtensionBase): def test_basic(self): module = self.import_extension('foo', [ @@ -31,7 +32,7 @@ if(s->ob_type->tp_basicsize != expected_size) { printf("tp_basicsize==%ld\\n", - (long)s->ob_type->tp_basicsize); + (long)s->ob_type->tp_basicsize); result = 0; } Py_DECREF(s); @@ -53,7 +54,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyByteArray_FromStringAndSize(NULL, 4); if (s == NULL) @@ -84,11 +84,10 @@ ("mutable", "METH_NOARGS", """ PyObject *base; - char * p_str; base = PyByteArray_FromStringAndSize("test", 10); if (PyByteArray_GET_SIZE(base) != 10) return PyLong_FromLong(-PyByteArray_GET_SIZE(base)); - memcpy(PyByteArray_AS_STRING(base), "works", 6); + memcpy(PyByteArray_AS_STRING(base), "works", 6); Py_INCREF(base); return base; """), @@ -115,6 +114,7 @@ assert s == 'test' def test_manipulations(self): + import sys module = self.import_extension('foo', [ ("bytearray_from_string", "METH_VARARGS", ''' @@ -141,9 +141,9 @@ ("concat", "METH_VARARGS", """ PyObject * ret, *right, *left; - PyObject *ba1, *ba2; + PyObject *ba1, *ba2; if (!PyArg_ParseTuple(args, "OO", &left, &right)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } ba1 = PyByteArray_FromObject(left); ba2 = PyByteArray_FromObject(right); @@ -157,7 +157,9 @@ """)]) assert module.bytearray_from_string("huheduwe") == "huhe" assert module.str_from_bytearray(bytearray('abc')) == 'abc' - raises(ValueError, module.str_from_bytearray, 4.0) + if '__pypy__' in sys.builtin_module_names: + # CPython only makes an assert. + raises(ValueError, module.str_from_bytearray, 4.0) ret = module.concat('abc', 'def') assert ret == 'abcdef' assert not isinstance(ret, str) @@ -171,9 +173,9 @@ PyObject *obj, *ba; int newsize, oldsize, ret; if (!PyArg_ParseTuple(args, "Oi", &obj, &newsize)) { - return PyString_FromString("parse failed"); + return PyString_FromString("parse failed"); } - + ba = PyByteArray_FromObject(obj); if (ba == NULL) return NULL; @@ -187,7 +189,7 @@ { printf("ret, oldsize, newsize= %d, %d, %d\\n", ret, oldsize, newsize); return NULL; - } + } return ba; ''' )]) diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -25,14 +25,13 @@ ("test_Size", "METH_NOARGS", """ PyObject* s = PyString_FromString("Hello world"); - int result = 0; + int result; size_t expected_size; - if(PyString_Size(s) == 11) { - result = 1; - } + result = PyString_Size(s); + #ifdef PYPY_VERSION - expected_size = sizeof(void*)*7; + expected_size = 48; #elif defined Py_DEBUG expected_size = 53; #else @@ -40,16 +39,16 @@ #endif if(s->ob_type->tp_basicsize != expected_size) { - printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); + printf("tp_basicsize==%zd\\n", s->ob_type->tp_basicsize); result = 0; } Py_DECREF(s); - return PyBool_FromLong(result); + return PyLong_FromLong(result); """), ("test_Size_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyString_Size(f); + PyString_Size(f); Py_DECREF(f); return NULL; @@ -60,7 +59,7 @@ """)], prologue='#include <stdlib.h>') assert module.get_hello1() == 'Hello world' assert module.get_hello2() == 'Hello world' - assert module.test_Size() + assert module.test_Size() == 11 raises(TypeError, module.test_Size_exception) assert module.test_is_string("") @@ -72,7 +71,6 @@ """ PyObject *s, *t; char* c; - Py_ssize_t len; s = PyString_FromStringAndSize(NULL, 4); if (s == NULL) @@ -81,7 +79,7 @@ if (t == NULL) return NULL; Py_DECREF(t); - c = PyString_AsString(s); + c = PyString_AS_STRING(s); c[0] = 'a'; c[1] = 'b'; c[2] = 0; @@ -100,7 +98,6 @@ PyObject *base; PyTypeObject * type; PyStringObject *obj; - char * p_str; base = PyString_FromString("test"); if (PyString_GET_SIZE(base) != 4) return PyLong_FromLong(-PyString_GET_SIZE(base)); @@ -110,14 +107,22 @@ obj = (PyStringObject*)type->tp_alloc(type, 10); if (PyString_GET_SIZE(obj) != 10) return PyLong_FromLong(PyString_GET_SIZE(obj)); - /* cannot work, there is only RO access - memcpy(PyString_AS_STRING(obj), "works", 6); */ + /* cannot work, there is only RO access */ + memcpy(PyString_AS_STRING(obj), "works", 6); Py_INCREF(obj); return (PyObject*)obj; """), + ('alloc_rw', "METH_NOARGS", + ''' + PyObject *obj = _PyObject_NewVar(&PyString_Type, 10); + memcpy(PyString_AS_STRING(obj), "works", 6); + return (PyObject*)obj; + '''), ]) + s = module.alloc_rw() + assert s == 'works' + '\x00' * 5 s = module.tpalloc() - assert s == '\x00' * 10 + assert s == 'works' + '\x00' * 5 def test_AsString(self): module = self.import_extension('foo', [ @@ -312,17 +317,17 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyStringObject*)obj)->ob_shash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ("test_sstate", "METH_NOARGS", ''' PyObject *s = PyString_FromString("xyz"); - int sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*int sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ PyString_InternInPlace(&s); - sstate = ((PyStringObject*)s)->ob_sstate; - /*printf("sstate now %d\\n", sstate);*/ + /*sstate = ((PyStringObject*)s)->ob_sstate; + printf("sstate now %d\\n", sstate);*/ Py_DECREF(s); return PyBool_FromLong(1); '''), @@ -332,27 +337,124 @@ # doesn't really test, but if printf is enabled will prove sstate assert module.test_sstate() + def test_subclass(self): + # taken from PyStringArrType_Type in numpy's scalartypes.c.src + module = self.import_extension('bar', [ + ("newsubstr", "METH_O", + """ + PyObject * obj; + char * data; + int len; + PyType_Ready(&PyStringArrType_Type); + + data = PyString_AS_STRING(args); + len = PyString_GET_SIZE(args); + if (data == NULL || len < 1) + Py_RETURN_NONE; + obj = PyArray_Scalar(data, len); + return obj; + """), + ], prologue=""" + #include <Python.h> + PyTypeObject PyStringArrType_Type = { + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ + "bar.string_", /* tp_name*/ + sizeof(PyStringObject), /* tp_basicsize*/ + 0 /* tp_itemsize */ + }; + + static PyObject * + stringtype_repr(PyObject *self) + { + const char *dptr, *ip; + int len; + PyObject *new; + + ip = dptr = PyString_AS_STRING(self); + len = PyString_GET_SIZE(self); + dptr += len-1; + while(len > 0 && *dptr-- == 0) { + len--; + } + new = PyString_FromStringAndSize(ip, len); + if (new == NULL) { + return PyString_FromString(""); + } + return new; + } + + static PyObject * + stringtype_str(PyObject *self) + { + const char *dptr, *ip; + int len; + PyObject *new; + + ip = dptr = PyString_AS_STRING(self); + len = PyString_GET_SIZE(self); + dptr += len-1; + while(len > 0 && *dptr-- == 0) { + len--; + } + new = PyString_FromStringAndSize(ip, len); + if (new == NULL) { + return PyString_FromString(""); + } + return new; + } + + PyObject * + PyArray_Scalar(char *data, int n) + { + PyTypeObject *type = &PyStringArrType_Type; + PyObject *obj; + void *destptr; + int itemsize = n; + obj = type->tp_alloc(type, itemsize); + if (obj == NULL) { + return NULL; + } + destptr = PyString_AS_STRING(obj); + ((PyStringObject *)obj)->ob_shash = -1; + memcpy(destptr, data, itemsize); + return obj; + } + """, more_init = ''' + PyStringArrType_Type.tp_alloc = NULL; + PyStringArrType_Type.tp_free = NULL; + + PyStringArrType_Type.tp_repr = stringtype_repr; + PyStringArrType_Type.tp_str = stringtype_str; + PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE; + PyStringArrType_Type.tp_itemsize = sizeof(char); + PyStringArrType_Type.tp_base = &PyString_Type; + ''') + + a = module.newsubstr('abc') + assert type(a).__name__ == 'string_' + assert a == 'abc' class TestString(BaseApiTest): def test_string_resize(self, space, api): py_str = new_empty_str(space, 10) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - py_str.c_buffer[0] = 'a' - py_str.c_buffer[1] = 'b' - py_str.c_buffer[2] = 'c' + py_str.c_ob_sval[0] = 'a' + py_str.c_ob_sval[1] = 'b' + py_str.c_ob_sval[2] = 'c' ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 3) py_str = rffi.cast(PyStringObject, ar[0]) assert py_str.c_ob_size == 3 - assert py_str.c_buffer[1] == 'b' - assert py_str.c_buffer[3] == '\x00' + assert py_str.c_ob_sval[1] == 'b' + assert py_str.c_ob_sval[3] == '\x00' # the same for growing ar[0] = rffi.cast(PyObject, py_str) api._PyString_Resize(ar, 10) py_str = rffi.cast(PyStringObject, ar[0]) assert py_str.c_ob_size == 10 - assert py_str.c_buffer[1] == 'b' - assert py_str.c_buffer[10] == '\x00' + assert py_str.c_ob_sval[1] == 'b' + assert py_str.c_ob_sval[10] == '\x00' Py_DecRef(space, ar[0]) lltype.free(ar, flavor='raw') diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -7,8 +7,6 @@ from pypy import pypydir from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import lltype, ll2ctypes -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator import platform from rpython.translator.gensupp import uniquemodulename from rpython.tool.udir import udir from pypy.module.cpyext import api @@ -18,20 +16,7 @@ from rpython.tool import leakfinder from rpython.rlib import rawrefcount -def setup_module(module): - if os.name == 'nt': - # Do not open dreaded dialog box on segfault - import ctypes - SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN - old_err_mode = ctypes.windll.kernel32.GetErrorMode() - new_err_mode = old_err_mode | SEM_NOGPFAULTERRORBOX - ctypes.windll.kernel32.SetErrorMode(new_err_mode) - module.old_err_mode = old_err_mode - -def teardown_module(module): - if os.name == 'nt': - import ctypes - ctypes.windll.kernel32.SetErrorMode(module.old_err_mode) +from .support import c_compile @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): @@ -46,7 +31,30 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def compile_extension_module(space, modname, include_dirs=[], **kwds): +def convert_sources_to_files(sources, dirname): + files = [] + for i, source in enumerate(sources): + filename = dirname / ('source_%d.c' % i) + with filename.open('w') as f: + f.write(str(source)) + files.append(filename) + return files + +def create_so(modname, include_dirs, source_strings=None, source_files=None, + compile_extra=None, link_extra=None, libraries=None): + dirname = (udir/uniquemodulename('module')).ensure(dir=1) + if source_strings: + assert not source_files + files = convert_sources_to_files(source_strings, dirname) + source_files = files + soname = c_compile(source_files, outputfilename=str(dirname/modname), + compile_extra=compile_extra, link_extra=link_extra, + include_dirs=include_dirs, + libraries=libraries) + return soname + +def compile_extension_module(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -60,38 +68,36 @@ state = space.fromcache(State) api_library = state.api_lib if sys.platform == 'win32': - kwds["libraries"] = [api_library] + libraries = [api_library] # '%s' undefined; assuming extern returning int - kwds["compile_extra"] = ["/we4013"] + compile_extra = ["/we4013"] # prevent linking with PythonXX.lib w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] - kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + link_extra = ["/NODEFAULTLIB:Python%d%d.lib" % (space.int_w(w_maj), space.int_w(w_min))] - elif sys.platform == 'darwin': - kwds["link_files"] = [str(api_library + '.dylib')] else: - kwds["link_files"] = [str(api_library + '.so')] + libraries = [] if sys.platform.startswith('linux'): - kwds["compile_extra"]=["-Werror", "-g", "-O0"] - kwds["link_extra"]=["-g"] + compile_extra = ["-Werror", "-g", "-O0", "-fPIC"] + link_extra = ["-g"] + else: + compile_extra = link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs=api.include_dirs + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=api.include_dirs + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra, + libraries=libraries) from pypy.module.imp.importing import get_so_extension pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) soname.rename(pydname) return str(pydname) -def compile_extension_module_applevel(space, modname, include_dirs=[], **kwds): +def compile_extension_module_applevel(space, modname, include_dirs=[], + source_files=None, source_strings=None): """ Build an extension module and return the filename of the resulting native code file. @@ -103,24 +109,23 @@ build the module (so specify your source with one of those). """ if sys.platform == 'win32': - kwds["compile_extra"] = ["/we4013"] - kwds["link_extra"] = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] + compile_extra = ["/we4013"] + link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] elif sys.platform == 'darwin': + compile_extra = link_extra = None pass elif sys.platform.startswith('linux'): - kwds["compile_extra"]=["-O0", "-g","-Werror=implicit-function-declaration"] + compile_extra = [ + "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] + link_extra = None modname = modname.split('.')[-1] - eci = ExternalCompilationInfo( - include_dirs = [space.include_dir] + include_dirs, - **kwds - ) - eci = eci.convert_sources_to_files() - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - soname = platform.platform.compile( - [], eci, - outputfilename=str(dirname/modname), - standalone=False) + soname = create_so(modname, + include_dirs=[space.include_dir] + include_dirs, + source_files=source_files, + source_strings=source_strings, + compile_extra=compile_extra, + link_extra=link_extra) return str(soname) def freeze_refcnts(self): @@ -285,8 +290,8 @@ separate_module_sources = [] pydname = self.compile_extension_module( space, name, - separate_module_files=separate_module_files, - separate_module_sources=separate_module_sources) + source_files=separate_module_files, + source_strings=separate_module_sources) return space.wrap(pydname) @gateway.unwrap_spec(name=str, init='str_or_None', body=str, @@ -315,6 +320,11 @@ /* fix for cpython 2.7 Python.h if running tests with -A since pypy compiles with -fvisibility-hidden */ #undef PyMODINIT_FUNC + #ifdef __GNUC__ + # define RPY_EXPORTED extern __attribute__((visibility("default"))) + #else + # define RPY_EXPORTED extern __declspec(dllexport) + #endif #define PyMODINIT_FUNC RPY_EXPORTED void %(body)s @@ -326,16 +336,16 @@ """ % dict(name=name, init=init, body=body, PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' if PY_SSIZE_T_CLEAN else '') - kwds = dict(separate_module_sources=[code]) + kwds = dict(source_strings=[code]) else: assert not PY_SSIZE_T_CLEAN if filename is None: filename = name filename = py.path.local(pypydir) / 'module' \ / 'cpyext'/ 'test' / (filename + ".c") - kwds = dict(separate_module_files=[filename]) - kwds['include_dirs'] = include_dirs - mod = self.compile_extension_module(space, name, **kwds) + kwds = dict(source_files=[filename]) + mod = self.compile_extension_module(space, name, + include_dirs=include_dirs, **kwds) if load_it: if self.runappdirect: @@ -975,7 +985,7 @@ ('bar', 'METH_NOARGS', ''' /* reuse a name that is #defined in structmember.h */ - int RO; + int RO = 0; (void)RO; Py_RETURN_NONE; ''' ), diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -142,7 +142,7 @@ 2000, 6, 6, 6, 6, 6, 6, Py_None, PyDateTimeAPI->DateTimeType); """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.new_date() == datetime.date(2000, 6, 6) assert module.new_time() == datetime.time(6, 6, 6, 6) @@ -241,6 +241,9 @@ PyObject* obj = PyDelta_FromDSU(6, 6, 6); PyDateTime_Delta* delta = (PyDateTime_Delta*)obj; +#if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000 + // These macros are only defined in CPython 3.x and PyPy. + // See: http://bugs.python.org/issue13727 PyDateTime_DELTA_GET_DAYS(obj); PyDateTime_DELTA_GET_DAYS(delta); @@ -249,10 +252,10 @@ PyDateTime_DELTA_GET_MICROSECONDS(obj); PyDateTime_DELTA_GET_MICROSECONDS(delta); - +#endif return obj; """), - ]) + ], prologue='#include "datetime.h"\n') import datetime assert module.test_date_macros() == datetime.date(2000, 6, 6) assert module.test_datetime_macros() == datetime.datetime( diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test/test_frameobject.py --- a/pypy/module/cpyext/test/test_frameobject.py +++ b/pypy/module/cpyext/test/test_frameobject.py @@ -13,7 +13,7 @@ PyObject *empty_string = PyString_FromString(""); PyObject *empty_tuple = PyTuple_New(0); PyCodeObject *py_code; - PyFrameObject *py_frame; + PyFrameObject *py_frame = NULL; py_code = PyCode_New( 0, /*int argcount,*/ @@ -75,7 +75,7 @@ """ int check; PyObject *type, *value, *tb; - PyObject *ret = PyRun_String("XXX", Py_eval_input, + PyObject *ret = PyRun_String("XXX", Py_eval_input, Py_None, Py_None); if (ret) { Py_DECREF(ret); diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -149,7 +149,6 @@ pybuffer = self.import_parser( ''' Py_buffer buf1, buf2, buf3; - PyObject *result; if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) { return NULL; } diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py --- a/pypy/module/cpyext/test/test_ndarrayobject.py +++ b/pypy/module/cpyext/test/test_ndarrayobject.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.micronumpy.ndarray import W_NDimArray from pypy.module.micronumpy.descriptor import get_dtype_cache -import pypy.module.micronumpy.constants as NPY +import pypy.module.micronumpy.constants as NPY def scalar(space): dtype = get_dtype_cache(space).w_float64dtype @@ -237,7 +237,7 @@ except: skip('numpy not importable') else: - numpy_incl = os.path.abspath(os.path.dirname(__file__) + + numpy_incl = os.path.abspath(os.path.dirname(__file__) + '/../include/_numpypy') assert os.path.exists(numpy_incl) cls.w_numpy_include = cls.space.wrap([numpy_incl]) @@ -273,7 +273,7 @@ { /* Should have failed */ Py_DECREF(obj1); - return NULL; + return NULL; } return obj1; ''' @@ -300,14 +300,14 @@ ), ("test_DescrFromType", "METH_O", """ - Signed typenum = PyInt_AsLong(args); + long typenum = PyInt_AsLong(args); return PyArray_DescrFromType(typenum); """ ), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include <pypy_numpy.h> + #include <pypy_numpy.h> #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include <numpy/arrayobject.h> @@ -315,7 +315,7 @@ #define PyArray_FromObject _PyArray_FromObject #define PyArray_FromAny _PyArray_FromAny #endif - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -349,14 +349,14 @@ Py_INCREF(obj); return obj; '''), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include <pypy_numpy.h> + #include <pypy_numpy.h> #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include <numpy/arrayobject.h> - ''', + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); @@ -403,14 +403,14 @@ void *array_data[] = {NULL, NULL}; return PyUFunc_FromFuncAndDataAndSignature(funcs, array_data, types, 1, 1, 1, PyUFunc_None, - "float_3x3", - "a ufunc that tests a more complicated signature", + "float_3x3", + "a ufunc that tests a more complicated signature", 0, "(m,m)->(m,m)"); """), - ], include_dirs=self.numpy_include, + ], include_dirs=self.numpy_include, prologue=''' #ifdef PYPY_VERSION - #include <pypy_numpy.h> + #include <pypy_numpy.h> #endif #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include <numpy/arrayobject.h> @@ -480,7 +480,7 @@ res += +10; *((float *)args[1]) = res; }; - + ''', more_init = ''' #ifndef PYPY_VERSION import_array(); diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -127,12 +127,12 @@ test_compare(1, 2) test_compare(2, 2) test_compare('2', '1') - + w_i = space.wrap(1) assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1 assert api.PyErr_Occurred() is space.w_SystemError api.PyErr_Clear() - + def test_IsInstance(self, space, api): assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1 assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0 @@ -165,7 +165,7 @@ return File""") w_f = space.call_function(w_File) assert api.PyObject_AsFileDescriptor(w_f) == 42 - + def test_hash(self, space, api): assert api.PyObject_Hash(space.wrap(72)) == 72 assert api.PyObject_Hash(space.wrap(-1)) == -1 @@ -250,7 +250,7 @@ if (copy != orig) PyObject_Free(copy); PyObject_Free(orig); - return ret; + return ret; """)]) x = module.realloctest() assert x == 'hello world\x00' @@ -425,7 +425,6 @@ """ Py_buffer buf; PyObject *str = PyString_FromString("hello, world."); - PyObject *result; if (PyBuffer_FillInfo(&buf, str, PyString_AsString(str), 13, 1, PyBUF_WRITABLE)) { diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -1,5 +1,7 @@ +from pypy.conftest import option from pypy.module.cpyext.api import fopen, fclose, fwrite from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.object import Py_PRINT_RAW from rpython.rtyper.lltypesystem import rffi, lltype from rpython.tool.udir import udir @@ -111,3 +113,34 @@ out, err = capfd.readouterr() out = out.replace('\r\n', '\n') assert out == " 1 23\n" + + +class AppTestPyFile(AppTestCpythonExtensionBase): + + def setup_class(cls): + from rpython.tool.udir import udir + if option.runappdirect: + cls.w_udir = str(udir) + else: + cls.w_udir = cls.space.wrap(str(udir)) + + def test_file_tell(self): + module = self.import_extension('foo', [ + ("get_c_tell", "METH_O", + """ + FILE * fp = PyFile_AsFile(args); + if (fp == NULL) + return PyLong_FromLong(0); + return PyLong_FromLong(ftell(fp)); + """), + ]) + filename = self.udir + "/_test_file" + with open(filename, 'w') as fid: + fid.write('3' * 122) + with open(filename, 'r') as fid: + s = fid.read(80) + t_py = fid.tell() + assert t_py == 80 + t_c = module.get_c_tell(fid) + assert t_c == t_py + diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -130,7 +130,7 @@ module = self.import_extension('foo', [ ("run", "METH_NOARGS", """ - long prev, next; + long prev; PyObject *t = PyTuple_New(1); prev = Py_True->ob_refcnt; Py_INCREF(Py_True); diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,8 +5,6 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr -import sys - class AppTestTypeObject(AppTestCpythonExtensionBase): def test_typeobject(self): import sys @@ -737,7 +735,6 @@ """ IntLikeObject *intObj; int intval; - PyObject *name; if (!PyArg_ParseTuple(args, "i", &intval)) return NULL; @@ -897,7 +894,7 @@ module.footype("X", (object,), {}) def test_app_subclass_of_c_type(self): - # on cpython, the size changes (6 bytes added) + import sys module = self.import_module(name='foo') size = module.size_of_instances(module.fooType) class f1(object): @@ -907,7 +904,11 @@ class bar(f1, f2): pass assert bar.__base__ is f2 - assert module.size_of_instances(bar) == size + # On cpython, the size changes. + if '__pypy__' in sys.builtin_module_names: + assert module.size_of_instances(bar) == size + else: + assert module.size_of_instances(bar) >= size def test_app_cant_subclass_two_types(self): module = self.import_module(name='foo') @@ -1058,7 +1059,6 @@ module = self.import_extension('foo', [ ("getMetaClass", "METH_NOARGS", ''' - PyObject *obj; FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT; FooType_Type.tp_base = &PyType_Type; if (PyType_Ready(&FooType_Type) < 0) return NULL; diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -35,7 +35,7 @@ ("test_GetSize_exception", "METH_NOARGS", """ PyObject* f = PyFloat_FromDouble(1.0); - Py_ssize_t size = PyUnicode_GetSize(f); + PyUnicode_GetSize(f); Py_DECREF(f); return NULL; @@ -57,7 +57,6 @@ """ PyObject *s, *t; Py_UNICODE* c; - Py_ssize_t len; s = PyUnicode_FromUnicode(NULL, 4); if (s == NULL) @@ -84,7 +83,7 @@ ''' PyObject* obj = (PyTuple_GetItem(args, 0)); long hash = ((PyUnicodeObject*)obj)->hash; - return PyLong_FromLong(hash); + return PyLong_FromLong(hash); ''' ), ]) @@ -234,13 +233,13 @@ w_res = api.PyUnicode_AsUTF8String(w_u) assert space.type(w_res) is space.w_str assert space.unwrap(w_res) == 'sp\tm' - + def test_decode_utf8(self, space, api): u = rffi.str2charp(u'sp\x134m'.encode("utf-8")) w_u = api.PyUnicode_DecodeUTF8(u, 5, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == u'sp\x134m' - + w_u = api.PyUnicode_DecodeUTF8(u, 2, None) assert space.type(w_u) is space.w_unicode assert space.unwrap(w_u) == 'sp' @@ -405,7 +404,7 @@ ustr = "abcdef" w_ustr = space.wrap(ustr.decode("ascii")) result = api.PyUnicode_AsASCIIString(w_ustr) - + assert space.eq_w(space.wrap(ustr), result) w_ustr = space.wrap(u"abcd\xe9f") diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -142,6 +142,7 @@ ref = rffi.cast(PyTupleObject, ref) size = ref.c_ob_size if index < 0 or index >= size: + decref(space, py_obj) raise oefmt(space.w_IndexError, "tuple assignment index out of range") old_ref = ref.c_ob_item[index] ref.c_ob_item[index] = py_obj # consumes a reference diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -657,6 +657,8 @@ pto.c_tp_dealloc = llhelper( subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) + if space.is_w(w_type, space.w_str): + pto.c_tp_itemsize = 1 # buffer protocol setup_buffer_procs(space, w_type, pto) @@ -695,6 +697,8 @@ if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize + if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize: + pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
