Author: Matti Picus <[email protected]>
Branch:
Changeset: r89616:5b9b0f8a10c5
Date: 2017-01-16 19:15 +0200
http://bitbucket.org/pypy/pypy/changeset/5b9b0f8a10c5/
Log: merge missing-tp_new which improves support for app-level classes in
cpyext
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -108,3 +108,14 @@
.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419
.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033
+
+.. branch: missing-tp_new
+
+Improve mixing app-level classes in c-extensions, especially if the app-level
+class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect
+all the method slots to be filled with a function pointer, where app-level will
+search up the mro for an appropriate function at runtime. With this branch we
+now fill many more slots in the c-extenion type objects.
+Also fix for c-extension type that calls ``tp_hash`` during initialization
+(str, unicode types), and fix instantiating c-extension types from built-in
+classes by enforcing an order of instaniation.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
from rpython.rlib import rawrefcount
from rpython.rlib import rthread
from rpython.rlib.debug import fatalerror_notb
+from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
DEBUG_WRAPPER = True
@@ -900,6 +901,7 @@
retval = fatal_value
boxed_args = ()
tb = None
+ state = space.fromcache(State)
try:
if not we_are_translated() and DEBUG_WRAPPER:
print >>sys.stderr, callable,
@@ -918,7 +920,6 @@
boxed_args += (arg_conv, )
if pygilstate_ensure:
boxed_args += (args[-1], )
- state = space.fromcache(State)
try:
result = callable(space, *boxed_args)
if not we_are_translated() and DEBUG_WRAPPER:
@@ -964,6 +965,7 @@
except Exception as e:
unexpected_exception(pname, e, tb)
_restore_gil_state(pygilstate_release, gilstate, gil_release,
_gil_auto, tid)
+ state.check_and_raise_exception(always=True)
return fatal_value
assert lltype.typeOf(retval) == restype
@@ -1124,6 +1126,34 @@
setup_init_functions(eci, prefix)
return modulename.new(ext='')
+def attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i):
+ # Start at i but make sure all the base classes are already attached
+ from pypy.module.cpyext.pyobject import get_typedescr, make_ref
+ if i in attached_objs:
+ return
+ py_obj = static_pyobjs[i]
+ w_obj = static_objs_w[i]
+ w_base = None
+ # w_obj can be NotImplemented, which is not a W_TypeObject
+ if isinstance(w_obj, W_TypeObject):
+ bases_w = w_obj.bases_w
+ if bases_w:
+ w_base = find_best_base(bases_w)
+ if w_base:
+ try:
+ j = static_objs_w.index(w_base)
+ except ValueError:
+ j = -1
+ if j >=0 and j not in attached_objs:
+ attach_recursively(space, static_pyobjs, static_objs_w,
+ attached_objs, j)
+ w_type = space.type(w_obj)
+ typedescr = get_typedescr(w_type.layout.typedef)
+ py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
+ make_ref(space, w_type))
+ typedescr.attach(space, py_obj, w_obj)
+ attached_objs.append(i)
+
class StaticObjectBuilder(object):
def __init__(self):
@@ -1144,7 +1174,6 @@
def attach_all(self, space):
# this is RPython, called once in pypy-c when it imports cpyext
- from pypy.module.cpyext.pyobject import get_typedescr, make_ref
from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
from pypy.module.cpyext.pyobject import track_reference
#
@@ -1154,14 +1183,9 @@
track_reference(space, static_pyobjs[i], static_objs_w[i])
#
self.cpyext_type_init = []
+ attached_objs = []
for i in range(len(static_objs_w)):
- py_obj = static_pyobjs[i]
- w_obj = static_objs_w[i]
- w_type = space.type(w_obj)
- typedescr = get_typedescr(w_type.layout.typedef)
- py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
- make_ref(space, w_type))
- typedescr.attach(space, py_obj, w_obj)
+ attach_recursively(space, static_pyobjs, static_objs_w,
attached_objs, i)
cpyext_type_init = self.cpyext_type_init
self.cpyext_type_init = None
for pto, w_type in cpyext_type_init:
diff --git a/pypy/module/cpyext/bytesobject.py
b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -80,14 +80,18 @@
"""
py_str = rffi.cast(PyBytesObject, py_obj)
s = space.str_w(w_obj)
- if py_str.c_ob_size < len(s):
+ len_s = len(s)
+ if py_str.c_ob_size < len_s:
raise oefmt(space.w_ValueError,
"bytes_attach called on object with ob_size %d but trying to store
%d",
- py_str.c_ob_size, len(s))
+ py_str.c_ob_size, len_s)
with rffi.scoped_nonmovingbuffer(s) as s_ptr:
- rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
- py_str.c_ob_sval[len(s)] = '\0'
- py_str.c_ob_shash = space.hash_w(w_obj)
+ rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len_s)
+ py_str.c_ob_sval[len_s] = '\0'
+ # if py_obj has a tp_hash, this will try to call it, but the objects are
+ # not fully linked yet
+ #py_str.c_ob_shash = space.hash_w(w_obj)
+ py_str.c_ob_shash = space.hash_w(space.newbytes(s))
py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
def bytes_realize(space, py_obj):
@@ -100,7 +104,9 @@
w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
w_obj = space.allocate_instance(W_BytesObject, w_type)
w_obj.__init__(s)
- py_str.c_ob_shash = space.hash_w(w_obj)
+ # if py_obj has a tp_hash, this will try to call it but the object is
+ # not realized yet
+ py_str.c_ob_shash = space.hash_w(space.newbytes(s))
py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
track_reference(space, py_obj, w_obj)
return w_obj
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -18,6 +18,7 @@
from pypy.module.cpyext.pyerrors import PyErr_Occurred
from pypy.module.cpyext.memoryobject import fill_Py_buffer
from pypy.module.cpyext.state import State
+from pypy.module.cpyext import userslot
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.argument import Arguments
from rpython.rlib.buffer import Buffer
@@ -512,6 +513,10 @@
@specialize.memo()
def get_slot_tp_function(space, typedef, name):
+ """Return a description of the slot C function to use for the built-in
+ type for 'typedef'. The 'name' is the slot name. This is a memo
+ function that, after translation, returns one of a built-in finite set.
+ """
key = (typedef, name)
try:
return SLOTS[key]
@@ -551,6 +556,19 @@
return space.call_function(slot_fn, w_self)
handled = True
+ for tp_name, attr in [('tp_hash', '__hash__'),
+ ]:
+ if name == tp_name:
+ slot_fn = w_type.getdictvalue(space, attr)
+ if slot_fn is None:
+ return
+ @slot_function([PyObject], lltype.Signed, error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'),
typedef.name))
+ def slot_func(space, w_obj):
+ return space.int_w(space.call_function(slot_fn, w_obj))
+ handled = True
+
+
# binary functions
for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
('tp_as_number.c_nb_subtract', '__sub__'),
@@ -755,7 +773,7 @@
else:
assert False
- function = globals().get(FUNCTION, None)
+ function = getattr(userslot, FUNCTION or '!missing', None)
assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS
if FLAGS:
if wrapper is Ellipsis:
@@ -818,7 +836,7 @@
static slotdef slotdefs[] = {
SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc,
"x.__len__() <==> len(x)"),
- SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc,
+ SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc,
"x.__add__(y) <==> x+y"),
SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc,
"x.__mul__(n) <==> x*n"),
@@ -966,9 +984,7 @@
"x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS),
TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
- TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""),
- TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""),
- TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
+ TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
"x.__setattr__('name', value) <==> x.name = value"),
TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -8,6 +8,24 @@
return newType;
}
+PyObject * typ = NULL;
+PyObject* datetimetype_tp_new(PyTypeObject* metatype, PyObject* args,
PyObject* kwds)
+{
+ PyObject* newType;
+ if (typ == NULL)
+ {
+ PyErr_Format(PyExc_TypeError, "could not import datetime.datetime");
+ return NULL;
+ }
+ newType = ((PyTypeObject*)typ)->tp_new(metatype, args, kwds);
+ return newType;
+}
+
+void datetimetype_tp_dealloc(PyObject* self)
+{
+ return ((PyTypeObject*)typ)->tp_dealloc(self);
+}
+
#define BASEFLAGS Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_CHECKTYPES
PyTypeObject footype = {
@@ -58,6 +76,54 @@
/*tp_weaklist*/ 0
};
+PyTypeObject datetimetype = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ /*tp_name*/ "foo3.datetimetype",
+ /*tp_basicsize*/ sizeof(PyTypeObject),
+ /*tp_itemsize*/ 0,
+ /*tp_dealloc*/ datetimetype_tp_dealloc,
+ /*tp_print*/ 0,
+ /*tp_getattr*/ 0,
+ /*tp_setattr*/ 0,
+ /*tp_compare*/ 0,
+ /*tp_repr*/ 0,
+ /*tp_as_number*/ 0,
+ /*tp_as_sequence*/ 0,
+ /*tp_as_mapping*/ 0,
+ /*tp_hash*/ 0,
+ /*tp_call*/ 0,
+ /*tp_str*/ 0,
+ /*tp_getattro*/ 0,
+ /*tp_setattro*/ 0,
+ /*tp_as_buffer*/ 0,
+ /*tp_flags*/ BASEFLAGS,
+ /*tp_doc*/ 0,
+ /*tp_traverse*/ 0,
+ /*tp_clear*/ 0,
+ /*tp_richcompare*/ 0,
+ /*tp_weaklistoffset*/ 0,
+ /*tp_iter*/ 0,
+ /*tp_iternext*/ 0,
+ /*tp_methods*/ 0,
+ /*tp_members*/ 0,
+ /*tp_getset*/ 0,
+ /*tp_base*/ 0, // set to &PyType_Type in module init
function (why can it not be done here?)
+ /*tp_dict*/ 0,
+ /*tp_descr_get*/ 0,
+ /*tp_descr_set*/ 0,
+ /*tp_dictoffset*/ 0,
+ /*tp_init*/ 0,
+ /*tp_alloc*/ 0,
+ /*tp_new*/ datetimetype_tp_new,
+ /*tp_free*/ 0,
+ /*tp_is_gc*/ 0,
+ /*tp_bases*/ 0,
+ /*tp_mro*/ 0,
+ /*tp_cache*/ 0,
+ /*tp_subclasses*/ 0,
+ /*tp_weaklist*/ 0
+};
+
static PyMethodDef sbkMethods[] = {{NULL, NULL, 0, NULL}};
/* Initialize this module. */
@@ -69,6 +135,16 @@
initfoo3(void)
{
PyObject *mod, *d;
+ PyObject *module = NULL;
+ module = PyImport_ImportModule("datetime");
+ typ = PyObject_GetAttr(module, PyString_FromString("datetime"));
+ Py_DECREF(module);
+ if (!PyType_Check(typ)) {
+ PyErr_Format(PyExc_TypeError, "datetime.datetime is not a type
object");
+ return;
+ }
+ datetimetype.tp_base = (PyTypeObject*)typ;
+ PyType_Ready(&datetimetype);
footype.tp_base = &PyType_Type;
PyType_Ready(&footype);
mod = Py_InitModule("foo3", sbkMethods);
@@ -79,5 +155,7 @@
return;
if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
return;
+ if (PyDict_SetItemString(d, "datetimetype", (PyObject *)&datetimetype) < 0)
+ return;
Py_INCREF(&footype);
}
diff --git a/pypy/module/cpyext/test/test_bytesobject.py
b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -357,11 +357,15 @@
data = PyString_AS_STRING(args);
len = PyString_GET_SIZE(args);
- if (data == NULL || len < 1)
+ if (data == NULL)
Py_RETURN_NONE;
obj = PyArray_Scalar(data, len);
return obj;
"""),
+ ("get_len", "METH_O",
+ """
+ return PyLong_FromLong(PyObject_Size(args));
+ """),
], prologue="""
#include <Python.h>
PyTypeObject PyStringArrType_Type = {
@@ -437,12 +441,16 @@
PyStringArrType_Type.tp_flags =
Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
PyStringArrType_Type.tp_itemsize = sizeof(char);
PyStringArrType_Type.tp_base = &PyString_Type;
+ PyStringArrType_Type.tp_hash = PyString_Type.tp_hash;
if (PyType_Ready(&PyStringArrType_Type) < 0) INITERROR;
''')
a = module.newsubstr('abc')
assert type(a).__name__ == 'string_'
assert a == 'abc'
+ assert 3 == module.get_len(a)
+ b = module.newsubstr('')
+ assert 0 == module.get_len(b)
class TestBytes(BaseApiTest):
def test_bytes_resize(self, space):
diff --git a/pypy/module/cpyext/test/test_typeobject.py
b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -2,6 +2,7 @@
from rpython.rtyper.lltypesystem import rffi
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.api import generic_cpy_call
from pypy.module.cpyext.pyobject import make_ref, from_ref
from pypy.module.cpyext.typeobject import PyTypeObjectPtr
@@ -663,30 +664,6 @@
assert module.tp_init(list, x, ("hi",)) is None
assert x == ["h", "i"]
- def test_tp_str(self):
- module = self.import_extension('foo', [
- ("tp_str", "METH_VARARGS",
- '''
- PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args,
0);
- PyObject *obj = PyTuple_GET_ITEM(args, 1);
- if (!type->tp_str)
- {
- PyErr_SetNone(PyExc_ValueError);
- return NULL;
- }
- return type->tp_str(obj);
- '''
- )
- ])
- class C:
- def __str__(self):
- return "text"
- assert module.tp_str(type(C()), C()) == "text"
- class D(int):
- def __str__(self):
- return "more text"
- assert module.tp_str(int, D(42)) == "42"
-
def test_mp_ass_subscript(self):
module = self.import_extension('foo', [
("new_obj", "METH_NOARGS",
@@ -978,9 +955,12 @@
assert (d + a) == 5
assert pow(d,b) == 16
- def test_tp_new_in_subclass_of_type(self):
+ def test_tp_new_in_subclass(self):
+ import datetime
module = self.import_module(name='foo3')
module.footype("X", (object,), {})
+ a = module.datetimetype(1, 1, 1)
+ assert isinstance(a, module.datetimetype)
def test_app_subclass_of_c_type(self):
import sys
@@ -1165,7 +1145,6 @@
self._check_type_object(FooType)
class X(object):
__metaclass__ = FooType
- print repr(X)
X()
def test_multiple_inheritance3(self):
diff --git a/pypy/module/cpyext/test/test_userslots.py
b/pypy/module/cpyext/test/test_userslots.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -0,0 +1,208 @@
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.pyobject import make_ref, from_ref
+from pypy.module.cpyext.api import generic_cpy_call
+from pypy.module.cpyext.typeobject import PyTypeObjectPtr
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class TestAppLevelObject(BaseApiTest):
+ def test_nb_add_from_python(self, space, api):
+ w_date = space.appexec([], """():
+ class DateType(object):
+ def __add__(self, other):
+ return 'sum!'
+ return DateType()
+ """)
+ w_datetype = space.type(w_date)
+ py_date = make_ref(space, w_date)
+ py_datetype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_datetype))
+ assert py_datetype.c_tp_as_number
+ assert py_datetype.c_tp_as_number.c_nb_add
+ w_obj = generic_cpy_call(space, py_datetype.c_tp_as_number.c_nb_add,
+ py_date, py_date)
+ assert space.str_w(w_obj) == 'sum!'
+
+ def test_tp_new_from_python(self, space, api):
+ w_date = space.appexec([], """():
+ class Date(object):
+ def __new__(cls, year, month, day):
+ self = object.__new__(cls)
+ self.year = year
+ self.month = month
+ self.day = day
+ return self
+ return Date
+ """)
+ py_datetype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_date))
+ one = space.newint(1)
+ arg = space.newtuple([one, one, one])
+ # call w_date.__new__
+ w_obj = space.call_function(w_date, one, one, one)
+ w_year = space.getattr(w_obj, space.newbytes('year'))
+ assert space.int_w(w_year) == 1
+
+ w_obj = generic_cpy_call(space, py_datetype.c_tp_new, py_datetype,
+ arg, space.newdict({}))
+ w_year = space.getattr(w_obj, space.newbytes('year'))
+ assert space.int_w(w_year) == 1
+
+class AppTestUserSlots(AppTestCpythonExtensionBase):
+ def test_tp_hash_from_python(self):
+ # to see that the functions are being used,
+ # run pytest with -s
+ module = self.import_extension('foo', [
+ ("use_hash", "METH_O",
+ '''
+ long hash = args->ob_type->tp_hash(args);
+ return PyLong_FromLong(hash);
+ ''')])
+ class C(object):
+ def __hash__(self):
+ return -23
+ c = C()
+ # uses the userslot slot_tp_hash
+ ret = module.use_hash(C())
+ assert hash(c) == ret
+ # uses the slotdef renamed cpyext_tp_hash_int
+ ret = module.use_hash(3)
+ assert hash(3) == ret
+
+ def test_tp_str(self):
+ module = self.import_extension('foo', [
+ ("tp_str", "METH_VARARGS",
+ '''
+ PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args,
0);
+ PyObject *obj = PyTuple_GET_ITEM(args, 1);
+ if (!type->tp_str)
+ {
+ PyErr_SetString(PyExc_ValueError, "no tp_str");
+ return NULL;
+ }
+ return type->tp_str(obj);
+ '''
+ )
+ ])
+ class C:
+ def __str__(self):
+ return "text"
+ assert module.tp_str(type(C()), C()) == "text"
+ class D(int):
+ def __str__(self):
+ return "more text"
+ assert module.tp_str(int, D(42)) == "42"
+ class A(object):
+ pass
+ s = module.tp_str(type(A()), A())
+ assert 'A object' in s
+
+ def test_tp_deallocate(self):
+ module = self.import_extension('foo', [
+ ("get_cnt", "METH_NOARGS",
+ '''
+ return PyLong_FromLong(foocnt);
+ '''),
+ ("get__timestamp", "METH_NOARGS",
+ '''
+ PyObject * one = PyLong_FromLong(1);
+ PyObject * a = PyTuple_Pack(3, one, one, one);
+ PyObject * k = NULL;
+ obj = _Timestamp.tp_new(&_Timestamp, a, k);
+ Py_DECREF(one);
+ return obj;
+ '''),
+ ("get_timestamp", "METH_NOARGS",
+ '''
+ PyObject * one = PyLong_FromLong(1);
+ PyObject * a = PyTuple_Pack(3, one, one, one);
+ PyObject * k = NULL;
+ obj = Timestamp.tp_new(&Timestamp, a, k);
+ Py_DECREF(one);
+ return obj;
+ '''),
+ ], prologue='''
+ static int foocnt = 0;
+ static PyTypeObject* datetime_cls = NULL;
+ static PyObject * obj = NULL;
+ static PyObject*
+ _timestamp_new(PyTypeObject* t, PyObject* a, PyObject* k)
+ {
+ foocnt ++;
+ return datetime_cls->tp_new(t, a, k);
+ }
+
+ static PyObject*
+ timestamp_new(PyTypeObject* t, PyObject* a, PyObject* k)
+ {
+ return datetime_cls->tp_new(t, a, k);
+ }
+
+ static void
+ _timestamp_dealloc(PyObject *op)
+ {
+ foocnt --;
+ datetime_cls->tp_dealloc(op);
+ }
+
+
+ static PyTypeObject _Timestamp = {
+ PyObject_HEAD_INIT(NULL)
+ 0, /* ob_size */
+ "foo._Timestamp", /* tp_name*/
+ 0, /* tp_basicsize*/
+ 0, /* tp_itemsize */
+ _timestamp_dealloc /* tp_dealloc */
+ };
+ static PyTypeObject Timestamp = {
+ PyObject_HEAD_INIT(NULL)
+ 0, /* ob_size */
+ "foo.Timestamp", /* tp_name*/
+ 0, /* tp_basicsize*/
+ 0 /* tp_itemsize */
+ };
+ ''', more_init='''
+ PyObject * mod = PyImport_ImportModule("datetime");
+ if (mod == NULL) INITERROR;
+ PyObject * dt = PyString_FromString("datetime");
+ datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod, dt);
+ if (datetime_cls == NULL) INITERROR;
+ _Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+ _Timestamp.tp_base = datetime_cls;
+ _Timestamp.tp_new = _timestamp_new;
+ Py_DECREF(mod);
+ Py_DECREF(dt);
+ if (PyType_Ready(&_Timestamp) < 0) INITERROR;
+
+ Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+ Timestamp.tp_base = &_Timestamp;
+ Timestamp.tp_new = timestamp_new;
+ Timestamp.tp_dealloc = datetime_cls->tp_dealloc;
+ if (PyType_Ready(&Timestamp) < 0) INITERROR;
+ ''')
+ # _Timestamp has __new__, __del__ and
+ # inherits from datetime.datetime
+ # Timestamp has __new__, default __del__ (subtype_dealloc) and
+ # inherits from _Timestamp
+ import gc, sys
+ cnt = module.get_cnt()
+ assert cnt == 0
+ obj = module.get__timestamp() #_Timestamp
+ cnt = module.get_cnt()
+ assert cnt == 1
+ assert obj.year == 1
+ del obj
+ self.debug_collect()
+ cnt = module.get_cnt()
+ assert cnt == 0
+
+ obj = module.get_timestamp() #Timestamp
+ cnt = module.get_cnt()
+ assert cnt == 0
+ assert obj.year == 1
+ # XXX calling Timestamp.tp_dealloc which is subtype_dealloc
+ # causes infinite recursion
+ del obj
+ self.debug_collect()
+ cnt = module.get_cnt()
+ assert cnt == 0
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -1,7 +1,7 @@
import os
from rpython.rlib import jit
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
from rpython.rtyper.lltypesystem import rffi, lltype
from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
@@ -36,7 +36,7 @@
from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
-WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
+#WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
@@ -258,6 +258,7 @@
dict_w[name] = w_descr
i += 1
+missing_slots={}
def update_all_slots(space, w_type, pto):
# fill slots in pto
# Not very sure about it, but according to
@@ -265,22 +266,46 @@
# overwrite slots that are already set: these ones are probably
# coming from a parent C type.
- typedef = w_type.layout.typedef
+ if w_type.is_heaptype():
+ typedef = None
+ search_dict_w = w_type.dict_w
+ else:
+ typedef = w_type.layout.typedef
+ search_dict_w = None
+
for method_name, slot_name, slot_names, slot_apifunc in
slotdefs_for_tp_slots:
- w_descr = w_type.lookup(method_name)
- if w_descr is None:
- # XXX special case iternext
- continue
+ slot_func_helper = None
+ if search_dict_w is None:
+ # built-in types: expose as many slots as possible, even
+ # if it happens to come from some parent class
+ slot_apifunc = None # use get_slot_tp_function
+ else:
+ # For heaptypes, w_type.layout.typedef will be object's typedef,
and
+ # get_slot_tp_function will fail
+ w_descr = search_dict_w.get(method_name, None)
+ if w_descr:
+ # use the slot_apifunc (userslots) to lookup at runtime
+ pass
+ elif len(slot_names) ==1:
+ # 'inherit' from tp_base
+ slot_func_helper = getattr(pto.c_tp_base, slot_names[0])
+ else:
+ struct = getattr(pto.c_tp_base, slot_names[0])
+ if struct:
+ slot_func_helper = getattr(struct, slot_names[1])
- if slot_apifunc is None and typedef is not None:
- slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
- if not slot_apifunc:
- if WARN_ABOUT_MISSING_SLOT_FUNCTIONS:
- os.write(2,
- "%s defined by %s but no slot function defined!\n" % (
- method_name, w_type.getname(space)))
- continue
- slot_func_helper = slot_apifunc.get_llhelper(space)
+ if not slot_func_helper:
+ if typedef is not None:
+ if slot_apifunc is None:
+ slot_apifunc = get_slot_tp_function(space, typedef,
slot_name)
+ if not slot_apifunc:
+ if not we_are_translated():
+ if slot_name not in missing_slots:
+ missing_slots[slot_name] = w_type.getname(space)
+ print "missing slot %r/%r, discovered on %r" % (
+ method_name, slot_name, w_type.getname(space))
+ continue
+ slot_func_helper = slot_apifunc.get_llhelper(space)
# XXX special case wrapper-functions and use a "specific" slot func
@@ -308,6 +333,8 @@
STRUCT_TYPE = PySequenceMethods
elif slot_names[0] == 'c_tp_as_buffer':
STRUCT_TYPE = PyBufferProcs
+ elif slot_names[0] == 'c_tp_as_mapping':
+ STRUCT_TYPE = PyMappingMethods
else:
raise AssertionError(
"Structure not allocated: %s" % (slot_names[0],))
@@ -405,13 +432,6 @@
pto.c_tp_itemsize = base_pto.c_tp_itemsize
pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
- flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
- base_object_pyo = make_ref(space, space.w_object)
- base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
- if base_pto != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
- if not pto.c_tp_new:
- pto.c_tp_new = base_pto.c_tp_new
- Py_DecRef(space, base_object_pyo)
def check_descr(space, w_self, w_type):
if not space.isinstance_w(w_self, w_type):
@@ -511,9 +531,25 @@
pto = obj.c_ob_type
base = pto
this_func_ptr = llslot(space, subtype_dealloc)
+ w_obj = from_ref(space, rffi.cast(PyObject, base))
+ # This wrapper is created on a specific type, call it w_A.
+ # We wish to call the dealloc function from one of the base classes of w_A,
+ # the first of which is not this function itself.
+ # w_obj is an instance of w_A or one of its subclasses. So climb up the
+ # inheritance chain until base.c_tp_dealloc is exactly this_func, and then
+ # continue on up until they differ.
+ #print 'subtype_dealloc, start from', rffi.charp2str(base.c_tp_name)
+ while base.c_tp_dealloc != this_func_ptr:
+ base = base.c_tp_base
+ assert base
+ #print ' ne move to', rffi.charp2str(base.c_tp_name)
+ w_obj = from_ref(space, rffi.cast(PyObject, base))
while base.c_tp_dealloc == this_func_ptr:
base = base.c_tp_base
assert base
+ #print ' eq move to', rffi.charp2str(base.c_tp_name)
+ w_obj = from_ref(space, rffi.cast(PyObject, base))
+ #print ' end with', rffi.charp2str(base.c_tp_name)
dealloc = base.c_tp_dealloc
# XXX call tp_del if necessary
generic_cpy_call(space, dealloc, obj)
@@ -685,13 +721,6 @@
typedescr = get_typedescr(w_type.layout.typedef)
- # dealloc
- if space.gettypeobject(w_type.layout.typedef) is w_type:
- # only for the exact type, like 'space.w_tuple' or 'space.w_list'
- pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
- else:
- # for all subtypes, use subtype_dealloc()
- pto.c_tp_dealloc = llslot(space, subtype_dealloc)
if space.is_w(w_type, space.w_str):
pto.c_tp_itemsize = 1
elif space.is_w(w_type, space.w_tuple):
@@ -722,6 +751,17 @@
w_base = best_base(space, w_type.bases_w)
pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base))
+ # dealloc
+ if space.gettypeobject(w_type.layout.typedef) is w_type:
+ # only for the exact type, like 'space.w_tuple' or 'space.w_list'
+ pto.c_tp_dealloc = typedescr.get_dealloc().get_llhelper(space)
+ else:
+ # for all subtypes, use base's dealloc (requires sorting in attach_all)
+ pto.c_tp_dealloc = pto.c_tp_base.c_tp_dealloc
+ if not pto.c_tp_dealloc:
+ # strange, but happens (ABCMeta)
+ pto.c_tp_dealloc = llslot(space, subtype_dealloc)
+
if builder.cpyext_type_init is not None:
builder.cpyext_type_init.append((pto, w_type))
else:
@@ -735,11 +775,18 @@
if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
- # will be filled later on with the correct value
- # may not be 0
if space.is_w(w_type, space.w_object):
+ # will be filled later on with the correct value
+ # may not be 0
pto.c_tp_new = rffi.cast(newfunc, 1)
update_all_slots(space, w_type, pto)
+ if not pto.c_tp_new:
+ base_object_pyo = make_ref(space, space.w_object)
+ base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
+ flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
+ if pto.c_tp_base != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
+ pto.c_tp_new = pto.c_tp_base.c_tp_new
+ Py_DecRef(space, base_object_pyo)
pto.c_tp_flags |= Py_TPFLAGS_READY
return pto
diff --git a/pypy/module/cpyext/unicodeobject.py
b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -65,9 +65,10 @@
def unicode_attach(space, py_obj, w_obj, w_userdata=None):
"Fills a newly allocated PyUnicodeObject with a unicode string"
py_unicode = rffi.cast(PyUnicodeObject, py_obj)
- py_unicode.c_length = len(space.unicode_w(w_obj))
+ s = space.unicode_w(w_obj)
+ py_unicode.c_length = len(s)
py_unicode.c_str = lltype.nullptr(rffi.CWCHARP.TO)
- py_unicode.c_hash = space.hash_w(w_obj)
+ py_unicode.c_hash = space.hash_w(space.newunicode(s))
py_unicode.c_defenc = lltype.nullptr(PyObject.TO)
def unicode_realize(space, py_obj):
@@ -80,7 +81,7 @@
w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
w_obj.__init__(s)
- py_uni.c_hash = space.hash_w(w_obj)
+ py_uni.c_hash = space.hash_w(space.newunicode(s))
track_reference(space, py_obj, w_obj)
return w_obj
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/userslot.py
@@ -0,0 +1,112 @@
+"""
+These are the default implementation for type slots that we put
+in user-defined app-level Python classes, if the class implements
+the corresponding '__xxx__' special method. It should mostly just
+call back the general version of the space operation.
+
+This is only approximately correct. One problem is that some
+details are likely subtly wrong. Another problem is that we don't
+track changes to an app-level Python class (addition or removal of
+'__xxx__' special methods) after initalization of the PyTypeObject.
+"""
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.module.cpyext.api import slot_function, PyObject, Py_ssize_t
+from pypy.module.cpyext.api import PyTypeObjectPtr
+from rpython.rtyper.lltypesystem import rffi, lltype
+
+@slot_function([PyObject], Py_ssize_t, error=-1)
+def slot_sq_length(space, w_obj):
+ return space.int_w(space.len(w_obj))
+
+@slot_function([PyObject], lltype.Signed, error=-1)
+def slot_tp_hash(space, w_obj):
+ return space.int_w(space.hash(w_obj))
+
+@slot_function([PyObject, Py_ssize_t], PyObject)
+def slot_sq_item(space, w_obj, index):
+ return space.getitem(w_obj, space.wrap(index))
+
+@slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
+def slot_tp_new(space, w_type, w_args, w_kwds):
+ # XXX problem - we need to find the actual __new__ function to call.
+ # but we have no 'self' argument. Theoretically, self will be
+ # w_type, but if w_type is a subclass of self, and w_type has a
+ # __new__ function that calls super().__new__, and that call ends
+ # up here, we will get infinite recursion. Prevent the recursion
+ # in the simple case (from cython) where w_type is a cpytype, but
+ # we know (since we are in this function) that self is not a cpytype
+ from pypy.module.cpyext.typeobject import W_PyCTypeObject
+ w_type0 = w_type
+ mro_w = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+ for w_m in mro_w[1:]:
+ if not w_type0.is_cpytype():
+ break
+ w_type0 = w_m
+ w_impl = space.getattr(w_type0, space.wrap('__new__'))
+ args = Arguments(space, [w_type],
+ w_stararg=w_args, w_starstararg=w_kwds)
+ return space.call_args(w_impl, args)
+
+# unary functions
+
+@slot_function([PyObject], PyObject)
+def slot_tp_str(space, w_obj):
+ return space.str(w_obj)
+
+@slot_function([PyObject], PyObject)
+def slot_tp_repr(space, w_obj):
+ return space.repr(w_obj)
+
+#binary functions
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_add(space, w_obj1, w_obj2):
+ return space.add(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_subtract(space, w_obj1, w_obj2):
+ return space.sub(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_multiply(space, w_obj1, w_obj2):
+ return space.mul(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_divide(space, w_obj1, w_obj2):
+ return space.div(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_add(space, w_obj1, w_obj2):
+ return space.add(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_subtract(space, w_obj1, w_obj2):
+ return space.sub(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_multiply(space, w_obj1, w_obj2):
+ return space.mul(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_nb_inplace_divide(space, w_obj1, w_obj2):
+ return space.div(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_sq_concat(space, w_obj1, w_obj2):
+ return space.add(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_sq_inplace_concat(space, w_obj1, w_obj2):
+ return space.add(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_mp_subscript(space, w_obj1, w_obj2):
+ return space.getitem(w_obj1, w_obj2)
+
+@slot_function([PyObject, PyObject], PyObject)
+def slot_tp_getattr(space, w_obj1, w_obj2):
+ return space.getattr(w_obj1, w_obj2)
+
+
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
