[pypy-commit] pypy cling-support: implement lookup of global operators

[wlav]

Author: Wim Lavrijsen 
Branch: cling-support
Changeset: r85717:14334ff3d976
Date: 2016-07-14 15:20 -0700
http://bitbucket.org/pypy/pypy/changeset/14334ff3d976/

Log:implement lookup of global operators

diff --git a/pypy/module/cppyy/interp_cppyy.py 
b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -1095,9 +1095,10 @@
 try:
 # TODO: expecting w_other to be an W_CPPInstance is too limiting
 other = self.space.interp_w(W_CPPInstance, w_other, 
can_be_None=False)
-for name in ["", "__gnu_cxx"]:
+for name in ["", "__gnu_cxx", "__1"]:
 nss = scope_byname(self.space, name)
-meth_idx = capi.c_get_global_operator(self.space, nss, 
self.cppclass, other.cppclass, "==")
+meth_idx = capi.c_get_global_operator(
+self.space, nss, self.cppclass, other.cppclass, 
"operator==")
 if meth_idx != -1:
 f = nss._make_cppfunction("operator==", meth_idx)
 ol = W_CPPOverload(self.space, nss, [f])
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx 
b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -112,10 +112,10 @@
 }
 
 static inline
-char* cppstring_to_cstring(const std::string& name) {
-char* name_char = (char*)malloc(name.size() + 1);
-strcpy(name_char, name.c_str());
-return name_char;
+char* cppstring_to_cstring( const std::string& name ) {
+   char* name_char = (char*)malloc(name.size() + 1 );
+   strcpy( name_char, name.c_str() );
+   return name_char;
 }
 
 
@@ -863,9 +863,23 @@
 }
 
 Cppyy::TCppIndex_t Cppyy::GetGlobalOperator(
-  TCppScope_t /* scope */, TCppType_t /* lc */, TCppType_t /* rc */, const 
std::string& /* op */ )
+  TCppScope_t scope, TCppType_t lc, TCppType_t rc, const std::string& 
opname )
 {
-   return (TCppIndex_t)0;
+// Find a global operator function with a matching signature
+   std::string proto = GetScopedFinalName(lc) + ", " + GetScopedFinalName(rc);
+   if ( scope == (cppyy_scope_t)GLOBAL_HANDLE ) {
+  TFunction* func = gROOT->GetGlobalFunctionWithPrototype( opname.c_str(), 
proto.c_str() );
+  if (func) return (TCppIndex_t)func;
+   } else {
+  TClassRef& cr = type_from_handle( scope );
+  if ( cr.GetClass() ) {
+ TFunction* func = cr->GetMethodWithPrototype( opname.c_str(), 
proto.c_str() );
+ if ( func ) return (TCppIndex_t)cr->GetListOfMethods()->IndexOf( func 
);
+  }
+   }
+
+// failure ...
+   return (TCppIndex_t)-1;
 }
 
 // method properties -
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cling-support: clean up some warnings about duplicates

[wlav]

Author: Wim Lavrijsen 
Branch: cling-support
Changeset: r85716:e6e59e1bc7fa
Date: 2016-07-14 15:09 -0700
http://bitbucket.org/pypy/pypy/changeset/e6e59e1bc7fa/

Log:clean up some warnings about duplicates

diff --git a/pypy/module/cppyy/test/stltypes.xml 
b/pypy/module/cppyy/test/stltypes.xml
--- a/pypy/module/cppyy/test/stltypes.xml
+++ b/pypy/module/cppyy/test/stltypes.xml
@@ -12,11 +12,11 @@
-->
 
   
-  
+  
   
 
-  
+  
 
   
   
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5-async: Create Coroutine class in generator with typedef (unfinished)

[raffael_t]

Author: Raffael Tfirst 
Branch: py3.5-async
Changeset: r85715:15b50bccd6cd
Date: 2016-07-14 23:13 +0200
http://bitbucket.org/pypy/pypy/changeset/15b50bccd6cd/

Log:Create Coroutine class in generator with typedef (unfinished)

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -304,6 +304,31 @@
 block = block.previous
 
 
+class Coroutine(W_Root):
+"A coroutine object."
+_immutable_fields_ = ['pycode']
+
+def __init__(self, frame):
+self.space = frame.space
+self.frame = frame # turned into None when frame_finished_execution
+self.pycode = frame.pycode
+self.running = False
+if self.pycode.co_flags & CO_YIELD_INSIDE_TRY:
+self.register_finalizer(self.space)
+
+def descr__repr__(self, space):
+if self.pycode is None:
+code_name = ''
+else:
+code_name = self.pycode.co_name
+addrstring = self.getaddrstring(space)
+return space.wrap("" %
+  (code_name, addrstring))
+
+#def close
+#w_close = space.getattr(w_yf, space.wrap("close"))
+
+
 def get_printable_location_genentry(bytecode):
 return '%s ' % (bytecode.get_repr(),)
 generatorentry_driver = jit.JitDriver(greens=['pycode'],
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1418,6 +1418,8 @@
 def GET_YIELD_FROM_ITER(self, oparg, next_instr):
 w_iterable = self.popvalue()
 w_iterator = self.space.iter(w_iterable)
+#ec = space.getexecutioncontext()
+#flags = ec.compiler.getcodeflags(self.pycode)
 self.pushvalue(w_iterator)
 
 def GET_AWAITABLE(self, oparg, next_instr):
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -466,7 +466,7 @@
 from pypy.interpreter.function import (Function, Method, StaticMethod,
 ClassMethod, BuiltinFunction, descr_function_get)
 from pypy.interpreter.pytraceback import PyTraceback
-from pypy.interpreter.generator import GeneratorIterator
+from pypy.interpreter.generator import GeneratorIterator, Coroutine
 from pypy.interpreter.nestedscope import Cell
 from pypy.interpreter.special import NotImplemented, Ellipsis
 
@@ -797,6 +797,28 @@
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
+Coroutine.typedef = TypeDef("coroutine",
+__repr__   = interp2app(GeneratorIterator.descr__repr__),
+__reduce__   = interp2app(GeneratorIterator.descr__reduce__),
+__setstate__ = interp2app(GeneratorIterator.descr__setstate__),
+__next__   = interp2app(GeneratorIterator.descr_next,
+descrmismatch='__next__'),
+send   = interp2app(GeneratorIterator.descr_send,
+descrmismatch='send'),
+throw  = interp2app(GeneratorIterator.descr_throw,
+descrmismatch='throw'),
+close  = interp2app(GeneratorIterator.descr_close,
+descrmismatch='close'),
+__iter__   = interp2app(GeneratorIterator.descr__iter__,
+descrmismatch='__iter__'),
+gi_running = interp_attrproperty('running', cls=GeneratorIterator),
+gi_frame   = GetSetProperty(GeneratorIterator.descr_gi_frame),
+gi_code= GetSetProperty(GeneratorIterator.descr_gi_code),
+__name__   = GetSetProperty(GeneratorIterator.descr__name__),
+__weakref__ = make_weakref_descr(GeneratorIterator),
+)
+assert not Coroutine.typedef.acceptable_as_base_class  # no __new__
+
 Cell.typedef = TypeDef("cell",
 __total_ordering__ = 'auto',
 __lt__   = interp2app(Cell.descr__lt__),
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy.org extradoc: update the values

[arigo]

Author: Armin Rigo 
Branch: extradoc
Changeset: r768:43bd6d95250b
Date: 2016-07-14 22:33 +0200
http://bitbucket.org/pypy/pypy.org/changeset/43bd6d95250b/

Log:update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 
   $(function() {
 $("#progressbar").progressbar({
-  value: 61.6
+  value: 61.7
});
   });
 
 

-   $64677 of $105000 (61.6%)
+   $64743 of $105000 (61.7%)


 
@@ -23,7 +23,7 @@
   
   This donation goes towards supporting Python 3 in 
PyPy.
   Current status:
-we have $6312 left
+we have $6372 left
   in the account. Read proposal
   
   
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 
   $(function() {
 $("#progressbar").progressbar({
-  value: 38.4
+  value: 38.5
});
   });
 
@@ -17,7 +17,7 @@
2nd call:

-   $30754 of $8 (38.4%)
+   $30764 of $8 (38.5%)


 
@@ -25,7 +25,7 @@
   
   This donation goes towards supporting the 
Transactional Memory in PyPy.
   Current status:
-we have $23425 left
+we have $23434 left
   in the account. Read proposal (2nd 
call)
   
   
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-slotdefs2: close branch to be merged

[mattip]

Author: Matti Picus 
Branch: cpyext-slotdefs2
Changeset: r85712:9016427866c2
Date: 2016-07-14 14:45 -0500
http://bitbucket.org/pypy/pypy/changeset/9016427866c2/

Log:close branch to be merged

___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: alternative phrasing

[mattip]

Author: Matti Picus 
Branch: 
Changeset: r85711:7bbaab81720f
Date: 2016-07-13 09:51 -0500
http://bitbucket.org/pypy/pypy/changeset/7bbaab81720f/

Log:alternative phrasing

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -380,7 +380,7 @@
   reproduced.  Or, maybe we can download a VirtualBox or VMWare
   virtual machine where the problem occurs.
 
-* If giving us access would require us to use other tools than ssh,
+* If giving us access would require us to use tools other than ssh,
   make appointments, or sign a NDA, then we can consider a commerical
   support contract for a small sum of money.
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: document merged branches

[mattip]

Author: Matti Picus 
Branch: 
Changeset: r85714:9540c130a102
Date: 2016-07-14 14:50 -0500
http://bitbucket.org/pypy/pypy/changeset/9540c130a102/

Log:document merged branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -72,3 +72,12 @@
 column might also not show the reduction, particularly on Linux >= 4.5 or
 on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
 the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-slotdefs2: fill more slots when creating a PyTypeObject from a W_TypeObject

[mattip]

Author: Matti Picus 
Branch: cpyext-slotdefs2
Changeset: r85707:345ec1c0c41e
Date: 2016-07-14 14:41 -0500
http://bitbucket.org/pypy/pypy/changeset/345ec1c0c41e/

Log:fill more slots when creating a PyTypeObject from a W_TypeObject

___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: merge cpyext-slotdefs2 which fills more slots when creating a PyTypeObject

[mattip]

Author: Matti Picus 
Branch: 
Changeset: r85713:ac0972f36db2
Date: 2016-07-14 14:45 -0500
http://bitbucket.org/pypy/pypy/changeset/ac0972f36db2/

Log:merge cpyext-slotdefs2 which fills more slots when creating a
PyTypeObject

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -380,6 +380,7 @@
   ('tp_as_number.c_nb_absolute', '__abs__'),
   ('tp_as_number.c_nb_invert', '__invert__'),
   ('tp_as_number.c_nb_index', '__index__'),
+  ('tp_as_number.c_nb_hex', '__hex__'),
   ('tp_str', '__str__'),
   ('tp_repr', '__repr__'),
   ('tp_iter', '__iter__'),
@@ -398,7 +399,7 @@
 
 # binary functions
 for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
-  ('tp_as_number.c_nb_subtract', '__subtract__'),
+  ('tp_as_number.c_nb_subtract', '__sub__'),
   ('tp_as_number.c_nb_multiply', '__mul__'),
   ('tp_as_number.c_nb_divide', '__div__'),
   ('tp_as_number.c_nb_remainder', '__mod__'),
@@ -408,6 +409,8 @@
   ('tp_as_number.c_nb_and', '__and__'),
   ('tp_as_number.c_nb_xor', '__xor__'),
   ('tp_as_number.c_nb_or', '__or__'),
+  ('tp_as_sequence.c_sq_concat', '__add__'),
+  ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
   ]:
 if name == tp_name:
 slot_fn = w_type.getdictvalue(space, attr)
@@ -421,8 +424,26 @@
 api_func = slot_func.api_func
 handled = True
 
+# binary-with-Py_ssize_t-type
+for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+  ('tp_as_sequence.c_sq_repeat', '__mul__'),
+  ('tp_as_sequence.c_sq_repeat', '__mul__'),
+  ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
+  ]:
+if name == tp_name:
+slot_fn = w_type.getdictvalue(space, attr)
+if slot_fn is None:
+return
+
+@cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
+@func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), 
typedef.name))
+def slot_func(space, w_self, arg):
+return space.call_function(slot_fn, w_self, space.wrap(arg))
+api_func = slot_func.api_func
+handled = True
+
 # ternary functions
-for tp_name, attr in [('tp_as_number.c_nb_power', ''),
+for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'),
   ]:
 if name == tp_name:
 slot_fn = w_type.getdictvalue(space, attr)
@@ -522,6 +543,8 @@
 api_func = slot_tp_new.api_func
 else:
 # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
+# tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
+# richcmpfunc(s)
 return
 
 return lambda: llhelper(api_func.functype, api_func.get_wrapper(space))
diff --git a/pypy/module/cpyext/test/test_listobject.py 
b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -64,7 +64,7 @@
 assert space.unwrap(w_s) == [2, 1]
 
 class AppTestListObject(AppTestCpythonExtensionBase):
-def test_listobject(self):
+def test_basic_listobject(self):
 import sys
 module = self.import_extension('foo', [
 ("newlist", "METH_NOARGS",
@@ -104,6 +104,15 @@
  Py_RETURN_NONE;
  """
  ),
+('test_tp_as_', "METH_NOARGS",
+ '''
+   PyObject *l = PyList_New(3);
+   int ok = l->ob_type->tp_as_sequence != NULL; /* 1 */
+   ok += 2 * (l->ob_type->tp_as_number == NULL); /* 2 */
+   Py_DECREF(l);
+   return PyLong_FromLong(ok); /* should be 3 */
+ '''
+ ),
 ])
 l = module.newlist()
 assert l == [3, -5, 1000]
@@ -137,6 +146,9 @@
 module.setlistitem(l,0)
 assert l == [None, 2, 3]
 
+# tp_as_sequence should be filled, but tp_as_number should be NULL
+assert module.test_tp_as_() == 3
+
 def test_list_macros(self):
 """The PyList_* macros cast, and calls expecting that build."""
 module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_longobject.py 
b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -227,4 +227,40 @@
  """)])
  

[pypy-commit] pypy cpyext-slotdefs2: hack to skip filling tp_as_number on (list, tuple). Needs eventual refactoring

[mattip]

Author: Matti Picus 
Branch: cpyext-slotdefs2
Changeset: r85709:61a414d6be12
Date: 2016-07-13 09:46 -0500
http://bitbucket.org/pypy/pypy/changeset/61a414d6be12/

Log:hack to skip filling tp_as_number on (list, tuple). Needs eventual
refactoring

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -238,7 +238,7 @@
 i += 1
 
 def update_all_slots(space, w_type, pto):
-#  XXX fill slots in pto
+# fill slots in pto
 # Not very sure about it, but according to
 # test_call_tp_dealloc_when_created_from_python, we should not
 # overwrite slots that are already set: these ones are probably
@@ -272,6 +272,15 @@
 if len(slot_names) == 1:
 if not getattr(pto, slot_names[0]):
 setattr(pto, slot_names[0], slot_func_helper)
+elif (w_type.getname(space) in ('list', 'tuple') and 
+  slot_names[0] == 'c_tp_as_number'):
+# XXX hack - hwo can we generalize this? The problem is method
+# names like __mul__ map to more than one slot, and we have no
+# convenient way to indicate which slots CPython have filled
+# 
+# We need at least this special case since Numpy checks that
+# (list, tuple) do __not__ fill tp_as_number
+pass
 else:
 assert len(slot_names) == 2
 struct = getattr(pto, slot_names[0])
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-slotdefs2: test, fix for __hex__, __sub__, __pow__ slots. More still missing

[mattip]

Author: Matti Picus 
Branch: cpyext-slotdefs2
Changeset: r85710:78f9ec6ff96f
Date: 2016-07-14 07:36 -0500
http://bitbucket.org/pypy/pypy/changeset/78f9ec6ff96f/

Log:test, fix for __hex__, __sub__, __pow__ slots. More still missing

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -380,6 +380,7 @@
   ('tp_as_number.c_nb_absolute', '__abs__'),
   ('tp_as_number.c_nb_invert', '__invert__'),
   ('tp_as_number.c_nb_index', '__index__'),
+  ('tp_as_number.c_nb_hex', '__hex__'),
   ('tp_str', '__str__'),
   ('tp_repr', '__repr__'),
   ('tp_iter', '__iter__'),
@@ -398,7 +399,7 @@
 
 # binary functions
 for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
-  ('tp_as_number.c_nb_subtract', '__subtract__'),
+  ('tp_as_number.c_nb_subtract', '__sub__'),
   ('tp_as_number.c_nb_multiply', '__mul__'),
   ('tp_as_number.c_nb_divide', '__div__'),
   ('tp_as_number.c_nb_remainder', '__mod__'),
@@ -426,6 +427,7 @@
 # binary-with-Py_ssize_t-type
 for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
   ('tp_as_sequence.c_sq_repeat', '__mul__'),
+  ('tp_as_sequence.c_sq_repeat', '__mul__'),
   ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
   ]:
 if name == tp_name:
@@ -441,7 +443,7 @@
 handled = True
 
 # ternary functions
-for tp_name, attr in [('tp_as_number.c_nb_power', ''),
+for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'),
   ]:
 if name == tp_name:
 slot_fn = w_type.getdictvalue(space, attr)
@@ -542,6 +544,7 @@
 else:
 # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
 # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
+# richcmpfunc(s)
 return
 
 return lambda: llhelper(api_func.functype, api_func.get_wrapper(space))
diff --git a/pypy/module/cpyext/test/test_longobject.py 
b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -227,4 +227,40 @@
  """)])
 assert module.from_str() == 0
 
-
+def test_slots(self):
+module = self.import_extension('foo', [
+("has_sub", "METH_NOARGS",
+ """
+PyObject *ret, *obj = PyLong_FromLong(42);
+if (obj->ob_type->tp_as_number->nb_subtract)
+ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+else
+ret = PyLong_FromLong(-1);
+Py_DECREF(obj);
+return ret;
+ """),
+ ("has_pow", "METH_NOARGS",
+ """
+PyObject *ret, *obj = PyLong_FromLong(42);
+PyObject *one = PyLong_FromLong(1);
+if (obj->ob_type->tp_as_number->nb_power)
+ret = obj->ob_type->tp_as_number->nb_power(obj, one, one);
+else
+ret = PyLong_FromLong(-1);
+Py_DECREF(obj);
+return ret;
+ """),
+("has_hex", "METH_NOARGS",
+ """
+PyObject *ret, *obj = PyLong_FromLong(42);
+if (obj->ob_type->tp_as_number->nb_hex)
+ret = obj->ob_type->tp_as_number->nb_hex(obj);
+else
+ret = PyLong_FromLong(-1);
+Py_DECREF(obj);
+return ret;
+ """)])
+assert module.has_sub() == 0
+assert module.has_pow() == 0
+assert module.has_hex() == '0x2aL'
+
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-slotdefs2: test, add tp_as_sequence slots when creating PyType_Object from PyPy

[mattip]

Author: Matti Picus 
Branch: cpyext-slotdefs2
Changeset: r85708:58297846176c
Date: 2016-07-13 09:27 -0500
http://bitbucket.org/pypy/pypy/changeset/58297846176c/

Log:test, add tp_as_sequence slots when creating PyType_Object from PyPy

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -408,6 +408,8 @@
   ('tp_as_number.c_nb_and', '__and__'),
   ('tp_as_number.c_nb_xor', '__xor__'),
   ('tp_as_number.c_nb_or', '__or__'),
+  ('tp_as_sequence.c_sq_concat', '__add__'),
+  ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
   ]:
 if name == tp_name:
 slot_fn = w_type.getdictvalue(space, attr)
@@ -421,6 +423,23 @@
 api_func = slot_func.api_func
 handled = True
 
+# binary-with-Py_ssize_t-type
+for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+  ('tp_as_sequence.c_sq_repeat', '__mul__'),
+  ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
+  ]:
+if name == tp_name:
+slot_fn = w_type.getdictvalue(space, attr)
+if slot_fn is None:
+return
+
+@cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
+@func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), 
typedef.name))
+def slot_func(space, w_self, arg):
+return space.call_function(slot_fn, w_self, space.wrap(arg))
+api_func = slot_func.api_func
+handled = True
+
 # ternary functions
 for tp_name, attr in [('tp_as_number.c_nb_power', ''),
   ]:
@@ -522,6 +541,7 @@
 api_func = slot_tp_new.api_func
 else:
 # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
+# tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
 return
 
 return lambda: llhelper(api_func.functype, api_func.get_wrapper(space))
diff --git a/pypy/module/cpyext/test/test_listobject.py 
b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -64,7 +64,7 @@
 assert space.unwrap(w_s) == [2, 1]
 
 class AppTestListObject(AppTestCpythonExtensionBase):
-def test_listobject(self):
+def test_basic_listobject(self):
 import sys
 module = self.import_extension('foo', [
 ("newlist", "METH_NOARGS",
@@ -104,6 +104,15 @@
  Py_RETURN_NONE;
  """
  ),
+('test_tp_as_', "METH_NOARGS",
+ '''
+   PyObject *l = PyList_New(3);
+   int ok = l->ob_type->tp_as_sequence != NULL; /* 1 */
+   ok += 2 * (l->ob_type->tp_as_number == NULL); /* 2 */
+   Py_DECREF(l);
+   return PyLong_FromLong(ok); /* should be 3 */
+ '''
+ ),
 ])
 l = module.newlist()
 assert l == [3, -5, 1000]
@@ -137,6 +146,9 @@
 module.setlistitem(l,0)
 assert l == [None, 2, 3]
 
+# tp_as_sequence should be filled, but tp_as_number should be NULL
+assert module.test_tp_as_() == 3
+
 def test_list_macros(self):
 """The PyList_* macros cast, and calls expecting that build."""
 module = self.import_extension('foo', [
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: merged default head

[plan_rich]

Author: Richard Plangger 
Branch: 
Changeset: r85706:0cdc78795c00
Date: 2016-07-14 13:49 -0500
http://bitbucket.org/pypy/pypy/changeset/0cdc78795c00/

Log:merged default head

diff --git a/pypy/module/_pypyjson/interp_decoder.py 
b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -360,10 +360,11 @@
 hexdigits = self.getslice(start, i)
 try:
 val = int(hexdigits, 16)
-if val & 0xfc00 == 0xd800:
+if sys.maxunicode > 65535 and 0xd800 <= val <= 0xdfff:
 # surrogate pair
-val = self.decode_surrogate_pair(i, val)
-i += 6
+if self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u':
+val = self.decode_surrogate_pair(i, val)
+i += 6
 except ValueError:
 self._raise("Invalid \u escape (char %d)", i-1)
 return # help the annotator to know that we'll never go beyond
@@ -375,8 +376,9 @@
 return i
 
 def decode_surrogate_pair(self, i, highsurr):
-if self.ll_chars[i] != '\\' or self.ll_chars[i+1] != 'u':
-self._raise("Unpaired high surrogate at char %d", i)
+""" uppon enter the following must hold:
+  chars[i] == "\\" and chars[i+1] == "u"
+"""
 i += 2
 hexdigits = self.getslice(i, i+4)
 lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by 
the caller
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py 
b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -184,6 +184,12 @@
 res = _pypyjson.loads('"z\\ud834\\udd20x"')
 assert res == expected
 
+def test_surrogate_pair(self):
+import _pypyjson
+json = '{"a":"\\uD83D"}'
+res = _pypyjson.loads(json)
+assert res == {u'a': u'\ud83d'}
+
 def test_tab_in_string_should_fail(self):
 import _pypyjson
 # http://json.org/JSON_checker/test/fail25.json
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy json-surrogates: close branch

[plan_rich]

Author: Richard Plangger 
Branch: json-surrogates
Changeset: r85704:409c2338bd70
Date: 2016-07-14 13:39 -0500
http://bitbucket.org/pypy/pypy/changeset/409c2338bd70/

Log:close branch

___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: merge fix for issue #2345

[plan_rich]

Author: Richard Plangger 
Branch: 
Changeset: r85705:fb7becc5ce15
Date: 2016-07-14 13:40 -0500
http://bitbucket.org/pypy/pypy/changeset/fb7becc5ce15/

Log:merge fix for issue #2345

diff --git a/pypy/module/_pypyjson/interp_decoder.py 
b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -360,10 +360,11 @@
 hexdigits = self.getslice(start, i)
 try:
 val = int(hexdigits, 16)
-if val & 0xfc00 == 0xd800:
+if sys.maxunicode > 65535 and 0xd800 <= val <= 0xdfff:
 # surrogate pair
-val = self.decode_surrogate_pair(i, val)
-i += 6
+if self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u':
+val = self.decode_surrogate_pair(i, val)
+i += 6
 except ValueError:
 self._raise("Invalid \u escape (char %d)", i-1)
 return # help the annotator to know that we'll never go beyond
@@ -375,8 +376,9 @@
 return i
 
 def decode_surrogate_pair(self, i, highsurr):
-if self.ll_chars[i] != '\\' or self.ll_chars[i+1] != 'u':
-self._raise("Unpaired high surrogate at char %d", i)
+""" uppon enter the following must hold:
+  chars[i] == "\\" and chars[i+1] == "u"
+"""
 i += 2
 hexdigits = self.getslice(i, i+4)
 lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by 
the caller
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py 
b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -184,6 +184,12 @@
 res = _pypyjson.loads('"z\\ud834\\udd20x"')
 assert res == expected
 
+def test_surrogate_pair(self):
+import _pypyjson
+json = '{"a":"\\uD83D"}'
+res = _pypyjson.loads(json)
+assert res == {u'a': u'\ud83d'}
+
 def test_tab_in_string_should_fail(self):
 import _pypyjson
 # http://json.org/JSON_checker/test/fail25.json
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cling-support: from Aditi: proper message on failure to load dictionary file

[wlav]

Author: Wim Lavrijsen 
Branch: cling-support
Changeset: r85701:6d4c1909c9ad
Date: 2016-07-14 09:30 -0700
http://bitbucket.org/pypy/pypy/changeset/6d4c1909c9ad/

Log:from Aditi: proper message on failure to load dictionary file

diff --git a/pypy/module/cppyy/interp_cppyy.py 
b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -32,6 +32,9 @@
 def load_dictionary(space, name):
 try:
 cdll = capi.c_load_dictionary(name)
+if not cdll:
+   raise OperationError(space.w_RuntimeError, space.wrap(str("could 
not load dictionary " + name)))
+
 except rdynload.DLOpenError as e:
 raise OperationError(space.w_RuntimeError, space.wrap(str(e.msg)))
 return W_CPPLibrary(space, cdll)
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx 
b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -20,6 +20,7 @@
 #include "TMethod.h"
 #include "TMethodArg.h"
 #include "TROOT.h"
+#include "TSystem.h"
 
 // Standard
 #include 
@@ -165,9 +166,10 @@
if ( icr != g_name2classrefidx.end() )
   return (TCppType_t)icr->second;
 
-   // use TClass directly, to enable auto-loading
+// use TClass directly, to enable auto-loading; class may be stubbed (eg. for
+// function returns) leading to a non-null TClass that is otherwise invalid
TClassRef cr( TClass::GetClass( scope_name.c_str(), kTRUE, kTRUE ) );
-   if ( !cr.GetClass() )
+   if ( !cr.GetClass() || !cr->Property() )
   return (TCppScope_t)NULL;
 
// no check for ClassInfo as forward declared classes are okay (fragile)
@@ -1390,7 +1392,8 @@
 /* misc helpers --- */
 RPY_EXTERN
 void* cppyy_load_dictionary(const char* lib_name) {
-return (void*)(gInterpreter->Load(lib_name) == 0);
+int result = gSystem->Load(lib_name);
+return (void*)(result == 0 /* success */ || result == 1 /* already loaded 
*/);
 }
 
 long long cppyy_strtoll(const char* str) {
diff --git a/pypy/module/cppyy/test/test_cppyy.py 
b/pypy/module/cppyy/test/test_cppyy.py
--- a/pypy/module/cppyy/test/test_cppyy.py
+++ b/pypy/module/cppyy/test/test_cppyy.py
@@ -4,7 +4,7 @@
 
 
 currpath = py.path.local(__file__).dirpath()
-test_dct = str(currpath.join("example01_rflx_rdict.pcm"))#example01Dict.so"))
+test_dct = str(currpath.join("example01Dict.so"))
 
 def setup_module(mod):
 if sys.platform == 'win32':
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cling-support: from Aditi: add signature strings

[wlav]

Author: Wim Lavrijsen 
Branch: cling-support
Changeset: r85702:60f4ffb2e3b4
Date: 2016-07-14 09:44 -0700
http://bitbucket.org/pypy/pypy/changeset/60f4ffb2e3b4/

Log:from Aditi: add signature strings

diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx 
b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -818,9 +818,22 @@
return "";
 }
 
-std::string Cppyy::GetMethodSignature( TCppScope_t /* scope */, TCppIndex_t /* 
imeth */ )
+std::string Cppyy::GetMethodSignature( TCppScope_t scope, TCppIndex_t imeth )
 {
-   return "";
+   TClassRef& cr = type_from_handle( scope );
+   TFunction* f = type_get_method( scope, imeth );
+   std::ostringstream sig;
+   if ( cr.GetClass() && cr->GetClassInfo() )
+  sig << f->GetReturnTypeName() << " ";
+   sig << cr.GetClassName() << "::" << f->GetName() << "(";
+   int nArgs = f->GetNargs();
+   for ( int iarg = 0; iarg < nArgs; ++iarg ) {
+  sig << ((TMethodArg*)f->GetListOfMethodArgs()->At( iarg 
))->GetFullTypeName();
+  if (iarg != nArgs-1)
+ sig << ", ";
+   }
+   sig << ")" << std::ends;
+   return cppstring_to_cstring(sig.str());
 }
 
 Bool_t Cppyy::IsConstMethod( TCppMethod_t method )
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cling-support: drop the use of g_globalfuncs vars as it is no longer needed for life-time management

[wlav]

Author: Wim Lavrijsen 
Branch: cling-support
Changeset: r85703:d24dadc47ff8
Date: 2016-07-14 11:09 -0700
http://bitbucket.org/pypy/pypy/changeset/d24dadc47ff8/

Log:drop the use of g_globalfuncs vars as it is no longer needed for
life-time management

diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx 
b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -48,9 +48,6 @@
 typedef std::map< Cppyy::TCppMethod_t, CallFunc_t* > Method2CallFunc_t;
 static Method2CallFunc_t g_method2callfunc;
 
-typedef std::vector< TFunction > GlobalFuncs_t;
-static GlobalFuncs_t g_globalfuncs;
-
 typedef std::vector< TGlobal* > GlobalVars_t;
 static GlobalVars_t g_globalvars;
 
@@ -699,24 +696,19 @@
 {
TClassRef& cr = type_from_handle (scope);
if (cr.GetClass())
-return (TCppIndex_t)imeth;
-assert(handle == (TCppType_t)GLOBAL_HANDLE);
-return (TCppIndex_t)_globalfuncs[imeth];
+  return (TCppIndex_t)imeth;
+   assert(handle == (TCppType_t)GLOBAL_HANDLE);
+   return imeth;
 }
 
 std::vector< Cppyy::TCppMethod_t > Cppyy::GetMethodsFromName(
   TCppScope_t scope, const std::string& name )
 {
-// TODO: this method assumes that the call for this name is made only
-// once, and thus there is no need to store the results of the search
-// in g_globalfuncs ... probably true, but needs verification
std::vector< TCppMethod_t > methods;
if ( scope == GLOBAL_HANDLE ) {
   TCollection* funcs = gROOT->GetListOfGlobalFunctions( kTRUE );
-  g_globalfuncs.reserve(funcs->GetSize());
 
   TIter ifunc(funcs);
-
   TFunction* func = 0;
   while ( (func = (TFunction*)ifunc.Next()) ) {
   // cover not only direct matches, but also template matches
@@ -822,18 +814,20 @@
 {
TClassRef& cr = type_from_handle( scope );
TFunction* f = type_get_method( scope, imeth );
-   std::ostringstream sig;
-   if ( cr.GetClass() && cr->GetClassInfo() )
-  sig << f->GetReturnTypeName() << " ";
-   sig << cr.GetClassName() << "::" << f->GetName() << "(";
-   int nArgs = f->GetNargs();
-   for ( int iarg = 0; iarg < nArgs; ++iarg ) {
-  sig << ((TMethodArg*)f->GetListOfMethodArgs()->At( iarg 
))->GetFullTypeName();
-  if (iarg != nArgs-1)
- sig << ", ";
+   if ( cr.GetClass() && cr->GetClassInfo() ) {
+  std::ostringstream sig;
+  sig << f->GetReturnTypeName() << " "
+  << cr.GetClassName() << "::" << f->GetName() << "(";
+  int nArgs = f->GetNargs();
+  for ( int iarg = 0; iarg < nArgs; ++iarg ) {
+ sig << ((TMethodArg*)f->GetListOfMethodArgs()->At( iarg 
))->GetFullTypeName();
+ if (iarg != nArgs-1)
+sig << ", ";
+  }
+  sig << ")" << std::ends;
+  return cppstring_to_cstring(sig.str());
}
-   sig << ")" << std::ends;
-   return cppstring_to_cstring(sig.str());
+   return "";
 }
 
 Bool_t Cppyy::IsConstMethod( TCppMethod_t method )
@@ -1246,23 +1240,23 @@
 }
 
 int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base){
-return (int)Cppyy::IsSubtype( derived, base );
+return (int)Cppyy::IsSubtype(derived, base);
 }
 
 
 /* calculate offsets between declared and actual type, up-cast: direction > 0; 
down-cast: direction < 0 */
 ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, 
cppyy_object_t address, int direction) {
-return (ptrdiff_t)Cppyy::GetBaseOffset(derived, base, (void*)address, 
direction,   0);
+return (ptrdiff_t)Cppyy::GetBaseOffset(derived, base, (void*)address, 
direction, 0);
 }
 
 
 /* method/function reflection information - */
 int cppyy_num_methods(cppyy_scope_t scope) {
-return (int)Cppyy::GetNumMethods (scope);
+return (int)Cppyy::GetNumMethods(scope);
 }
 
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth) {
-return cppyy_index_t(Cppyy::GetMethodIndexAt (scope, imeth));
+return cppyy_index_t(Cppyy::GetMethodIndexAt(scope, imeth));
 }
 
 cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* 
name){
@@ -1275,7 +1269,7 @@
 TIter next(cr->GetListOfMethods());
 while ((func = (TFunction*)next())) {
 if (strcmp(name, func->GetName()) == 0) {
-if (func->Property() & 
Cppyy::IsPublicMethod((cppyy_method_t)func))
+if (Cppyy::IsPublicMethod((cppyy_method_t)func))
 result.push_back((cppyy_index_t)imeth);
 }
 ++imeth;
@@ -1285,10 +1279,8 @@
 TFunction* func = 0;
 TIter ifunc(funcs);
 while ((func = (TFunction*)ifunc.Next())) {
-if (strcmp(name, func->GetName()) == 0) {
-g_globalfuncs.push_back(*func);
+if (strcmp(name, func->GetName()) == 0)
 result.push_back((cppyy_index_t)func);
-}
 }
 }
 
___
pypy-commit 

[pypy-commit] pypy json-surrogates: align json module decode with the cpython's impl

[plan_rich]

Author: Richard Plangger 
Branch: json-surrogates
Changeset: r85700:ec18acf52840
Date: 2016-07-14 11:43 -0500
http://bitbucket.org/pypy/pypy/changeset/ec18acf52840/

Log:align json module decode with the cpython's impl

diff --git a/pypy/module/_pypyjson/interp_decoder.py 
b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -360,10 +360,11 @@
 hexdigits = self.getslice(start, i)
 try:
 val = int(hexdigits, 16)
-if val & 0xfc00 == 0xd800:
+if sys.maxunicode > 65535 and 0xd800 <= val <= 0xdfff:
 # surrogate pair
-val = self.decode_surrogate_pair(i, val)
-i += 6
+if self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u':
+val = self.decode_surrogate_pair(i, val)
+i += 6
 except ValueError:
 self._raise("Invalid \u escape (char %d)", i-1)
 return # help the annotator to know that we'll never go beyond
@@ -375,8 +376,9 @@
 return i
 
 def decode_surrogate_pair(self, i, highsurr):
-if self.ll_chars[i] != '\\' or self.ll_chars[i+1] != 'u':
-self._raise("Unpaired high surrogate at char %d", i)
+""" uppon enter the following must hold:
+  chars[i] == "\\" and chars[i+1] == "u"
+"""
 i += 2
 hexdigits = self.getslice(i, i+4)
 lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by 
the caller
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy json-surrogates: add test to ensure to adapt the behaviour in the json module (see #2345)

[plan_rich]

Author: Richard Plangger 
Branch: json-surrogates
Changeset: r85699:e3ee9d867add
Date: 2016-07-14 11:13 -0500
http://bitbucket.org/pypy/pypy/changeset/e3ee9d867add/

Log:add test to ensure to adapt the behaviour in the json module (see
#2345)

diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py 
b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -184,6 +184,12 @@
 res = _pypyjson.loads('"z\\ud834\\udd20x"')
 assert res == expected
 
+def test_surrogate_pair(self):
+import _pypyjson
+json = '{"a":"\\uD83D"}'
+res = _pypyjson.loads(json)
+assert res == {u'a': u'\ud83d'}
+
 def test_tab_in_string_should_fail(self):
 import _pypyjson
 # http://json.org/JSON_checker/test/fail25.json
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy ppc-vsx-support: adds removed function to free fail args registers

[plan_rich]

Author: Richard Plangger 
Branch: ppc-vsx-support
Changeset: r85698:2954ca91b89a
Date: 2016-07-13 18:16 -0500
http://bitbucket.org/pypy/pypy/changeset/2954ca91b89a/

Log:adds removed function to free fail args registers

diff --git a/rpython/jit/backend/ppc/codebuilder.py 
b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -709,7 +709,7 @@
 lvsl = XV(31, XO1=6)
 lvsr = XV(31, XO1=38)
 vperm = VA(4, XO10=43)
-vsel = VA(4, XO10=42) 
+vsel = VA(4, XO10=42)
 vspltisb = VXI(4, XO8=780)
 vspltisw = VXI(4, XO8=844)
 vspltisw = VXI(4, XO8=908)
diff --git a/rpython/jit/backend/ppc/regalloc.py 
b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -585,6 +585,7 @@
 args.append(self.loc(arg))
 else:
 args.append(None)
+self.possibly_free_vars(op.getfailargs())
 #
 # generate_quick_failure() produces up to 14 instructions per guard
 self.limit_loop_break -= 14 * 4
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5-async: Apply cpython 3.5.1 change regarding async function flag

[raffael_t]

Author: Raffael Tfirst 
Branch: py3.5-async
Changeset: r85697:7db7039d9d40
Date: 2016-07-14 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/7db7039d9d40/

Log:Apply cpython 3.5.1 change regarding async function flag

diff --git a/pypy/interpreter/astcompiler/codegen.py 
b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -405,7 +405,7 @@
 oparg |= num_annotations << 16
 code, qualname = self.sub_scope(AsyncFunctionCodeGenerator, func.name, 
func,
 func.lineno)
-code.co_flags |= consts.CO_GENERATOR
+code.co_flags |= consts.CO_COROUTINE
 self._make_function(code, oparg, qualname=qualname)
 # Apply decorators.
 if func.decorator_list:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5-async: set CO_GENERATOR flag in async function code

[raffael_t]

Author: Raffael Tfirst 
Branch: py3.5-async
Changeset: r85696:43805b5f2878
Date: 2016-07-14 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/43805b5f2878/

Log:set CO_GENERATOR flag in async function code

diff --git a/pypy/interpreter/astcompiler/codegen.py 
b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -405,6 +405,7 @@
 oparg |= num_annotations << 16
 code, qualname = self.sub_scope(AsyncFunctionCodeGenerator, func.name, 
func,
 func.lineno)
+code.co_flags |= consts.CO_GENERATOR
 self._make_function(code, oparg, qualname=qualname)
 # Apply decorators.
 if func.decorator_list:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy issue2343: Copy CPython's logic more closely

[arigo]

Author: Armin Rigo 
Branch: issue2343
Changeset: r85695:66c0e2134605
Date: 2016-07-14 16:54 +0200
http://bitbucket.org/pypy/pypy/changeset/66c0e2134605/

Log:Copy CPython's logic more closely

diff --git a/pypy/module/__builtin__/abstractinst.py 
b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -46,72 +46,34 @@
 raise   # propagate other errors
 return space.type(w_obj)
 
-@jit.unroll_safe
-def abstract_isinstance_w(space, w_obj, w_klass_or_tuple, 
allow_override=False):
-"""Implementation for the full 'isinstance(obj, klass_or_tuple)'."""
-# -- case (anything, tuple)
-# XXX it might be risky that the JIT sees this
-if space.isinstance_w(w_klass_or_tuple, space.w_tuple):
-for w_klass in space.fixedview(w_klass_or_tuple):
-if abstract_isinstance_w(space, w_obj, w_klass, allow_override):
-return True
-return False
 
-# -- case (anything, type)
+# -- isinstance --
+
+
+def p_recursive_isinstance_w(space, w_inst, w_cls):
+# Copied straight from CPython 2.7.  Does not handle 'cls' being a tuple.
+if (isinstance(w_cls, W_ClassObject) and
+isinstance(w_inst, W_InstanceObject)):
+return w_inst.w_class.is_subclass_of(w_cls)
+
+if space.isinstance_w(w_cls, space.w_type):
+return p_recursive_isinstance_type_w(space, w_inst, w_cls)
+
+check_class(space, w_cls, "isinstance() arg 2 must be a class, type,"
+  " or tuple of classes and types")
 try:
-if allow_override:
-w_result = space.isinstance_allow_override(w_obj, w_klass_or_tuple)
-else:
-w_result = space.isinstance(w_obj, w_klass_or_tuple)
-except OperationError as e:   # if w_klass_or_tuple was not a type, ignore 
it
-if not e.match(space, space.w_TypeError):
-raise   # propagate other errors
-else:
-if space.is_true(w_result):
-return True
-# From now on we know that w_klass_or_tuple is indeed a type.
-# Try also to compare it with obj.__class__, if this is not
-# the same as type(obj).
-try:
-w_pretendtype = space.getattr(w_obj, space.wrap('__class__'))
-if space.is_w(w_pretendtype, space.type(w_obj)):
-return False # common case: obj.__class__ is type(obj)
-if not allow_override:
-return space.issubtype_w(w_pretendtype, w_klass_or_tuple)
-w_result = space.issubtype_allow_override(w_pretendtype,
-  w_klass_or_tuple)
-except OperationError as e:
-if e.async(space):
-raise
-return False  # ignore most exceptions
-else:
-return space.is_true(w_result)
-
-# -- case (old-style instance, old-style class)
-if isinstance(w_klass_or_tuple, W_ClassObject):
-if isinstance(w_obj, W_InstanceObject):
-return w_obj.w_class.is_subclass_of(w_klass_or_tuple)
-return _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple)
-
-
-def _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple):
-# -- case (anything, abstract-class)
-check_class(space, w_klass_or_tuple,
-"isinstance() arg 2 must be a class, type,"
-" or tuple of classes and types")
-try:
-w_abstractclass = space.getattr(w_obj, space.wrap('__class__'))
+w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
 except OperationError as e:
 if e.async(space):  # ignore most exceptions
 raise
 return False
 else:
-return _issubclass_recurse(space, w_abstractclass, w_klass_or_tuple)
+return p_abstract_issubclass_w(space, w_abstractclass, w_cls)
 
 
-def recursive_isinstance_type_w(space, w_inst, w_type):
-# subfunctionality of recursive_isinstance(): assumes that w_type is
-# a type object.
+def p_recursive_isinstance_type_w(space, w_inst, w_type):
+# subfunctionality of p_recursive_isinstance_w(): assumes that w_type is
+# a type object.  Copied straight from CPython 2.7.
 if space.isinstance_w(w_inst, w_type):
 return True
 try:
@@ -126,9 +88,73 @@
 return False
 
 
-def recursive_issubclass_w(space, w_derived, w_cls):
-# From CPython's function of the same name, which as far as I can tell
-# is not recursive.
+@jit.unroll_safe
+def abstract_isinstance_w(space, w_obj, w_klass_or_tuple, 
allow_override=False):
+"""Implementation for the full 'isinstance(obj, klass_or_tuple)'."""
+# Copied from CPython 2.7's PyObject_Isinstance().  Additionally,
+# if 'allow_override' is False (the default), then don't try to
+# use a custom __instancecheck__ method.
+
+# WARNING: backward compatibility 

[pypy-commit] pypy issue2343: in-progress, trying to copy the logic of CPython and finding messes

[arigo]

Author: Armin Rigo 
Branch: issue2343
Changeset: r85694:d581b89514a5
Date: 2016-07-14 15:55 +0200
http://bitbucket.org/pypy/pypy/changeset/d581b89514a5/

Log:in-progress, trying to copy the logic of CPython and finding messes
in our different logic

diff --git a/pypy/module/__builtin__/abstractinst.py 
b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -109,6 +109,41 @@
 return _issubclass_recurse(space, w_abstractclass, w_klass_or_tuple)
 
 
+def recursive_isinstance_type_w(space, w_inst, w_type):
+# subfunctionality of recursive_isinstance(): assumes that w_type is
+# a type object.
+if space.isinstance_w(w_inst, w_type):
+return True
+try:
+w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+except OperationError as e:
+if e.async(space):  # ignore most exceptions
+raise
+else:
+if w_abstractclass is not space.type(w_inst):
+if space.isinstance_w(w_abstractclass, space.w_type):
+return space.issubtype_w(w_abstractclass, w_type)
+return False
+
+
+def recursive_issubclass_w(space, w_derived, w_cls):
+# From CPython's function of the same name, which as far as I can tell
+# is not recursive.
+if (space.isinstance_w(w_cls, space.w_type) and
+space.isinstance_w(w_derived, space.w_type)):
+return space.issubtype_w(w_derived, w_cls)
+#
+if (isinstance(w_derived, W_ClassObject) and
+isinstance(w_cls, W_ClassObject)):
+return w_derived.is_subclass_of(w_cls)
+#
+check_class(space, w_derived, "issubclass() arg 1 must be a class")
+check_class(space, w_cls, "issubclass() arg 2 must be a class"
+  " or tuple of classes")
+
+return abstract_issubclass_w(space, w_derived, w_cls)
+
+
 @jit.unroll_safe
 def _issubclass_recurse(space, w_derived, w_top):
 """Internal helper for abstract cases.  Here, w_top cannot be a tuple."""
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\
  descr_get_dict, dict_descr, Member, TypeDef
 from pypy.interpreter.astcompiler.misc import mangle
+from pypy.module.__builtin__ import abstractinst
 
 from rpython.rlib.jit import (promote, elidable_promote, we_are_jitted,
  elidable, dont_look_inside, unroll_safe)
@@ -899,13 +900,15 @@
 
 # 
 
-@gateway.unwrap_spec(w_obj=W_TypeObject, w_sub=W_TypeObject)
+@gateway.unwrap_spec(w_obj=W_TypeObject)
 def type_issubtype(w_obj, space, w_sub):
-return space.newbool(w_sub.issubtype(w_obj))
+return space.newbool(
+abstractinst.recursive_issubclass_w(space, w_sub, w_obj))
 
 @gateway.unwrap_spec(w_obj=W_TypeObject)
 def type_isinstance(w_obj, space, w_inst):
-return space.newbool(space.type(w_inst).issubtype(w_obj))
+return space.newbool(
+abstractinst.recursive_isinstance_type_w(space, w_inst, w_obj))
 
 W_TypeObject.typedef = TypeDef("type",
 __new__ = gateway.interp2app(descr__new__),
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy issue2343: fix test

[arigo]

Author: Armin Rigo 
Branch: issue2343
Changeset: r85693:1009a35a5711
Date: 2016-07-14 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/1009a35a5711/

Log:fix test

diff --git a/pypy/module/__builtin__/test/test_abstractinst.py 
b/pypy/module/__builtin__/test/test_abstractinst.py
--- a/pypy/module/__builtin__/test/test_abstractinst.py
+++ b/pypy/module/__builtin__/test/test_abstractinst.py
@@ -238,4 +238,4 @@
 def __subclasscheck__(self, subcls):
 return False
 
-assert issubclass(42, ()) is False
+assert issubclass(42, M()) is False
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5-async: Add Coroutine flags

[raffael_t]

Author: Raffael Tfirst 
Branch: py3.5-async
Changeset: r85692:a48e02697dd8
Date: 2016-07-14 14:29 +0200
http://bitbucket.org/pypy/pypy/changeset/a48e02697dd8/

Log:Add Coroutine flags

diff --git a/pypy/interpreter/astcompiler/consts.py 
b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -9,6 +9,8 @@
 CO_NESTED = 0x0010
 CO_GENERATOR = 0x0020
 CO_NOFREE = 0x0040
+CO_COROUTINE = 0x0080
+CO_ITERABLE_COROUTINE = 0x0100
 CO_GENERATOR_ALLOWED = 0x1000
 CO_FUTURE_DIVISION = 0x2000
 CO_FUTURE_ABSOLUTE_IMPORT = 0x4000
@@ -16,6 +18,7 @@
 CO_FUTURE_PRINT_FUNCTION = 0x1
 CO_FUTURE_UNICODE_LITERALS = 0x2
 CO_FUTURE_BARRY_AS_BDFL = 0x4
+CO_FUTURE_GENERATOR_STOP = 0x8
 #pypy specific:
 CO_KILL_DOCSTRING = 0x10
 CO_YIELD_INSIDE_TRY = 0x20
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy.org extradoc: Windows madvise_free() supported too

[arigo]

Author: Armin Rigo 
Branch: extradoc
Changeset: r767:547759aa9d78
Date: 2016-07-14 14:04 +0200
http://bitbucket.org/pypy/pypy.org/changeset/547759aa9d78/

Log:Windows madvise_free() supported too

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -125,11 +125,11 @@
 the program finishes running in the meantime).  See http://pypy.readthedocs.org/en/latest/cpython_differences.html#differences-related-to-garbage-collection-strategies;>more
 details
 here.
 Note that PyPy returns unused memory to the operating system if there
-is a madvise() system call (at least Linux, OS/X, BSD).  It is
+is a madvise() system call (at least Linux, OS/X, BSD) or on Windows.  It is
 important to realize that you may not see this in top.  The unused
 pages are marked with MADV_FREE, which tells 
the system if you
 need more memory at some point, grab this page.  As long as memory is
-plentiful, the RES column in top remains high.  (Exceptions to
+plentiful, the RES column in top might remains high.  (Exceptions to
 this rule are systems with no MADV_FREE, 
where we use
 MADV_DONTNEED, which forcefully lowers the 
RES.  This includes
 Linux = 4.4.)
diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -115,11 +115,11 @@
 here`_.
 
 Note that PyPy returns unused memory to the operating system if there
-is a madvise() system call (at least Linux, OS/X, BSD).  It is
+is a madvise() system call (at least Linux, OS/X, BSD) or on Windows.  It is
 important to realize that you may not see this in ``top``.  The unused
 pages are marked with ``MADV_FREE``, which tells the system "if you
 need more memory at some point, grab this page".  As long as memory is
-plentiful, the ``RES`` column in ``top`` remains high.  (Exceptions to
+plentiful, the ``RES`` column in ``top`` might remains high.  (Exceptions to
 this rule are systems with no ``MADV_FREE``, where we use
 ``MADV_DONTNEED``, which forcefully lowers the ``RES``.  This includes
 Linux <= 4.4.)
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: Windows support for use-madv-free

[arigo]

Author: Armin Rigo 
Branch: 
Changeset: r85691:06f4abd385c4
Date: 2016-07-14 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/06f4abd385c4/

Log:Windows support for use-madv-free

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -82,7 +82,8 @@
 constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY',
   'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY',
   'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE',
-  'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS']
+  'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS',
+  'MEM_RESET']
 for name in constant_names:
 setattr(CConfig, name, rffi_platform.ConstantInteger(name))
 
@@ -232,6 +233,9 @@
 VirtualAlloc, VirtualAlloc_safe = winexternal('VirtualAlloc',
[rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD],
rffi.VOIDP)
+_, _VirtualAlloc_safe_no_wrapper = winexternal('VirtualAlloc',
+   [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD],
+   rffi.VOIDP, _nowrapper=True)
 _, _VirtualProtect_safe = winexternal('VirtualProtect',
   [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD],
   BOOL)
@@ -949,3 +953,12 @@
 
 def free(ptr, map_size):
 VirtualFree_safe(ptr, 0, MEM_RELEASE)
+
+def madvise_free(addr, map_size):
+r = _VirtualAlloc_safe_no_wrapper(
+rffi.cast(rffi.VOIDP, addr),
+rffi.cast(rffi.SIZE_T, map_size),
+rffi.cast(DWORD, MEM_RESET),
+rffi.cast(DWORD, PAGE_READWRITE))
+from rpython.rlib import debug
+debug.debug_print("madvise_free:", r)
diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py
--- a/rpython/rlib/test/test_rmmap.py
+++ b/rpython/rlib/test/test_rmmap.py
@@ -5,11 +5,7 @@
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import rmmap as mmap
 from rpython.rlib.rmmap import RTypeError, RValueError, alloc, free
-try:
-from rpython.rlib.rmmap import madvise_free
-except ImportError:
-def madvise_free(addr, size):
-"Not available"
+from rpython.rlib.rmmap import madvise_free
 
 
 class TestMMap:
diff --git a/rpython/rtyper/lltypesystem/llarena.py 
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -440,17 +440,6 @@
 if size > 0:# clear the final misaligned part, if any
 llmemory.raw_memclear(baseaddr, size)
 
-def madvise_arena_free(baseaddr, size):
-from rpython.rlib import rmmap
-
-pagesize = posixpagesize.get()
-baseaddr = rffi.cast(lltype.Signed, baseaddr)
-aligned_addr = (baseaddr + pagesize - 1) & ~(pagesize - 1)
-size -= (aligned_addr - baseaddr)
-if size >= pagesize:
-rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr),
-   size & ~(pagesize - 1))
-
 else:
 # XXX any better implementation on Windows?
 # Should use VirtualAlloc() to reserve the range of pages,
@@ -459,10 +448,22 @@
 # them immediately.
 clear_large_memory_chunk = llmemory.raw_memclear
 
-def madvise_arena_free(baseaddr, size):
-"""XXX find a Windows equivalent?
-'baseaddr' is in the middle of memory obtained with the C malloc()...
-"""
+class PosixPageSize:
+def get(self):
+from rpython.rlib import rmmap
+return rmmap.PAGESIZE
+posixpagesize = PosixPageSize()
+
+def madvise_arena_free(baseaddr, size):
+from rpython.rlib import rmmap
+
+pagesize = posixpagesize.get()
+baseaddr = rffi.cast(lltype.Signed, baseaddr)
+aligned_addr = (baseaddr + pagesize - 1) & ~(pagesize - 1)
+size -= (aligned_addr - baseaddr)
+if size >= pagesize:
+rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr),
+   size & ~(pagesize - 1))
 
 
 if os.name == "posix":
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy.org extradoc: Document the memory usage behavior of PyPy, because it is not very clear

[arigo]

Author: Armin Rigo 
Branch: extradoc
Changeset: r766:c4d464e272a9
Date: 2016-07-14 12:37 +0200
http://bitbucket.org/pypy/pypy.org/changeset/c4d464e272a9/

Log:Document the memory usage behavior of PyPy, because it is not very
clear that memory usage *does* go down in systems with MADV_FREE.

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -124,6 +124,15 @@
 as on CPython: they run some time later in PyPy (or not at all if
 the program finishes running in the meantime).  See http://pypy.readthedocs.org/en/latest/cpython_differences.html#differences-related-to-garbage-collection-strategies;>more
 details
 here.
+Note that PyPy returns unused memory to the operating system if there
+is a madvise() system call (at least Linux, OS/X, BSD).  It is
+important to realize that you may not see this in top.  The unused
+pages are marked with MADV_FREE, which tells 
the system if you
+need more memory at some point, grab this page.  As long as memory is
+plentiful, the RES column in top remains high.  (Exceptions to
+this rule are systems with no MADV_FREE, 
where we use
+MADV_DONTNEED, which forcefully lowers the 
RES.  This includes
+Linux = 4.4.)
 A more complete list of known differences is available at http://pypy.readthedocs.org/en/latest/cpython_differences.html;>our dev 
site.
 
 
diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -114,6 +114,16 @@
 the program finishes running in the meantime).  See `more details
 here`_.
 
+Note that PyPy returns unused memory to the operating system if there
+is a madvise() system call (at least Linux, OS/X, BSD).  It is
+important to realize that you may not see this in ``top``.  The unused
+pages are marked with ``MADV_FREE``, which tells the system "if you
+need more memory at some point, grab this page".  As long as memory is
+plentiful, the ``RES`` column in ``top`` remains high.  (Exceptions to
+this rule are systems with no ``MADV_FREE``, where we use
+``MADV_DONTNEED``, which forcefully lowers the ``RES``.  This includes
+Linux <= 4.4.)
+
 A more complete list of known differences is available at `our dev site`_.
 
 .. _`CPython C API`: http://docs.python.org/c-api/
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: Document branch

[arigo]

Author: Armin Rigo 
Branch: 
Changeset: r85690:66bb0cd7b0eb
Date: 2016-07-14 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/66bb0cd7b0eb/

Log:Document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -64,3 +64,11 @@
 Avoid exhausting the stack in the JIT due to successive guard
 failures in the same Python function ending up as successive levels of
 RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy use-madv-free: Windows test fix

[arigo]

Author: Armin Rigo 
Branch: use-madv-free
Changeset: r85687:10d990c7213f
Date: 2016-07-14 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/10d990c7213f/

Log:Windows test fix

diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py
--- a/rpython/rlib/test/test_rmmap.py
+++ b/rpython/rlib/test/test_rmmap.py
@@ -5,7 +5,12 @@
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import rmmap as mmap
 from rpython.rlib.rmmap import RTypeError, RValueError, alloc, free
-from rpython.rlib.rmmap import madvise_free
+try:
+from rpython.rlib.rmmap import madvise_free
+except ImportError:
+def madvise_free(addr, size):
+"Not available"
+
 
 class TestMMap:
 def setup_class(cls):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy use-madv-free: ready to merge

[arigo]

Author: Armin Rigo 
Branch: use-madv-free
Changeset: r85688:4bd0dcbf94e3
Date: 2016-07-14 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/4bd0dcbf94e3/

Log:ready to merge

___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: hg merge use-madv-free

[arigo]

Author: Armin Rigo 
Branch: 
Changeset: r85689:95f85a57f5c9
Date: 2016-07-14 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/95f85a57f5c9/

Log:hg merge use-madv-free

Use madvise(MADV_FREE), or if that doesn't exist MADV_DONTNEED, on
freed arenas. Fixes issue #2336.

diff --git a/rpython/memory/gc/minimarkpage.py 
b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -395,6 +395,7 @@
 if arena.nfreepages == arena.totalpages:
 #
 # The whole arena is empty.  Free it.
+llarena.arena_reset(arena.base, self.arena_size, 4)
 llarena.arena_free(arena.base)
 lltype.free(arena, flavor='raw', track_allocation=False)
 #
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -70,7 +70,13 @@
 CConfig.MREMAP_MAYMOVE = (
 rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE"))
 CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)')
-# a dirty hack, this is probably a macro
+CConfig.has_madvise = rffi_platform.Has('madvise(NULL, 0, 0)')
+# ^^ both are a dirty hack, this is probably a macro
+
+CConfig.MADV_DONTNEED = (
+rffi_platform.DefinedConstantInteger('MADV_DONTNEED'))
+CConfig.MADV_FREE = (
+rffi_platform.DefinedConstantInteger('MADV_FREE'))
 
 elif _MS_WINDOWS:
 constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY',
@@ -144,6 +150,7 @@
 
 if _POSIX:
 has_mremap = cConfig['has_mremap']
+has_madvise = cConfig['has_madvise']
 c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT,
rffi.INT, off_t], PTR, macro=True,
save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO)
@@ -154,6 +161,9 @@
 if has_mremap:
 c_mremap, _ = external('mremap',
[PTR, size_t, size_t, rffi.ULONG], PTR)
+if has_madvise:
+_, c_madvise_safe = external('madvise', [PTR, size_t, rffi.INT],
+ rffi.INT, _nowrapper=True)
 
 # this one is always safe
 _pagesize = rffi_platform.getintegerfunctionresult('getpagesize',
@@ -755,6 +765,39 @@
 else:
 free = c_munmap_safe
 
+if sys.platform.startswith('linux'):
+assert has_madvise
+assert MADV_DONTNEED is not None
+if MADV_FREE is None:
+MADV_FREE = 8 # from the kernel sources of Linux >= 4.5
+class CanUseMadvFree:
+ok = -1
+can_use_madv_free = CanUseMadvFree()
+def madvise_free(addr, map_size):
+# We don't know if we are running on a recent enough kernel
+# that supports MADV_FREE.  Check that at runtime: if the
+# first call to madvise(MADV_FREE) fails, we assume it's
+# because of EINVAL and we fall back to MADV_DONTNEED.
+if can_use_madv_free.ok != 0:
+res = c_madvise_safe(rffi.cast(PTR, addr),
+ rffi.cast(size_t, map_size),
+ rffi.cast(rffi.INT, MADV_FREE))
+if can_use_madv_free.ok == -1:
+can_use_madv_free.ok = (rffi.cast(lltype.Signed, res) == 0)
+if can_use_madv_free.ok == 0:
+c_madvise_safe(rffi.cast(PTR, addr),
+   rffi.cast(size_t, map_size),
+   rffi.cast(rffi.INT, MADV_DONTNEED))
+elif has_madvise and not (MADV_FREE is MADV_DONTNEED is None):
+use_flag = MADV_FREE if MADV_FREE is not None else MADV_DONTNEED
+def madvise_free(addr, map_size):
+c_madvise_safe(rffi.cast(PTR, addr),
+   rffi.cast(size_t, map_size),
+   rffi.cast(rffi.INT, use_flag))
+else:
+def madvice_free(addr, map_size):
+"No madvice() on this platform"
+
 elif _MS_WINDOWS:
 def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
 # XXX flags is or-ed into access by now.
@@ -906,5 +949,3 @@
 
 def free(ptr, map_size):
 VirtualFree_safe(ptr, 0, MEM_RELEASE)
-
-# register_external here?
diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py
--- a/rpython/rlib/test/test_rmmap.py
+++ b/rpython/rlib/test/test_rmmap.py
@@ -5,6 +5,12 @@
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import rmmap as mmap
 from rpython.rlib.rmmap import RTypeError, RValueError, alloc, free
+try:
+from rpython.rlib.rmmap import madvise_free
+except ImportError:
+def madvise_free(addr, size):
+"Not available"
+
 
 class TestMMap:
 def setup_class(cls):
@@ -490,6 +496,7 @@
 data[i] = chr(i & 0xff)
 for i in range(0, map_size, 

[pypy-commit] pypy use-madv-free: Test and fix

[arigo]

Author: Armin Rigo 
Branch: use-madv-free
Changeset: r85686:83477b04d5ed
Date: 2016-07-14 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/83477b04d5ed/

Log:Test and fix

diff --git a/rpython/rtyper/lltypesystem/llarena.py 
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -445,11 +445,11 @@
 
 pagesize = posixpagesize.get()
 baseaddr = rffi.cast(lltype.Signed, baseaddr)
-aligned_addr = (baseaddr + pagesize - 1) & (pagesize - 1)
+aligned_addr = (baseaddr + pagesize - 1) & ~(pagesize - 1)
 size -= (aligned_addr - baseaddr)
 if size >= pagesize:
 rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr),
-   size & (pagesize - 1))
+   size & ~(pagesize - 1))
 
 else:
 # XXX any better implementation on Windows?
diff --git a/rpython/rtyper/lltypesystem/test/test_llarena.py 
b/rpython/rtyper/lltypesystem/test/test_llarena.py
--- a/rpython/rtyper/lltypesystem/test/test_llarena.py
+++ b/rpython/rtyper/lltypesystem/test/test_llarena.py
@@ -1,6 +1,6 @@
-import py
+import py, os
 
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llarena
 from rpython.rtyper.lltypesystem.llarena import (arena_malloc, arena_reset,
 arena_reserve, arena_free, round_up_for_allocation, ArenaError,
 arena_new_view, arena_shrink_obj, arena_protect, has_protect)
@@ -299,6 +299,29 @@
 p.x = 125
 assert p.x == 125
 
+def test_madvise_arena_free():
+from rpython.rlib import rmmap
+
+if os.name != 'posix':
+py.test.skip("posix only")
+pagesize = llarena.posixpagesize.get()
+prev = rmmap.madvise_free
+try:
+seen = []
+def my_madvise_free(addr, size):
+assert lltype.typeOf(addr) == rmmap.PTR
+seen.append((addr, size))
+rmmap.madvise_free = my_madvise_free
+llarena.madvise_arena_free(
+rffi.cast(llmemory.Address, 123 * pagesize + 1),
+pagesize * 7 - 2)
+finally:
+rmmap.madvise_free = prev
+assert len(seen) == 1
+addr, size = seen[0]
+assert rffi.cast(lltype.Signed, addr) == 124 * pagesize
+assert size == pagesize * 5
+
 
 class TestStandalone(test_standalone.StandaloneTests):
 def test_compiled_arena_protect(self):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy use-madv-free: Call madvise() from minimarkpage and llarena

[arigo]

Author: Armin Rigo 
Branch: use-madv-free
Changeset: r85685:78c7dacab7ab
Date: 2016-07-14 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/78c7dacab7ab/

Log:Call madvise() from minimarkpage and llarena

diff --git a/rpython/memory/gc/minimarkpage.py 
b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -395,6 +395,7 @@
 if arena.nfreepages == arena.totalpages:
 #
 # The whole arena is empty.  Free it.
+llarena.arena_reset(arena.base, self.arena_size, 4)
 llarena.arena_free(arena.base)
 lltype.free(arena, flavor='raw', track_allocation=False)
 #
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -949,10 +949,3 @@
 
 def free(ptr, map_size):
 VirtualFree_safe(ptr, 0, MEM_RELEASE)
-
-def madvice_free(addr, map_size):
-"""XXX find a Windows equivalent?
-'addr' is in the middle of memory obtained with the C malloc()...
-"""
-
-# register_external here?
diff --git a/rpython/rtyper/lltypesystem/llarena.py 
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -52,7 +52,7 @@
 del self.objectptrs[offset]
 del self.objectsizes[offset]
 obj._free()
-if zero and zero != 3:
+if zero in (1, 2):
 initialbyte = "0"
 else:
 initialbyte = "#"
@@ -335,6 +335,8 @@
   * 1: clear, optimized for a very large area of memory
   * 2: clear, optimized for a small or medium area of memory
   * 3: fill with garbage
+  * 4: large area of memory that can benefit from MADV_FREE
+ (i.e. contains garbage, may be zero-filled or not)
 """
 arena_addr = getfakearenaaddress(arena_addr)
 arena_addr.arena.reset(zero, arena_addr.offset, size)
@@ -410,16 +412,19 @@
 self.pagesize = 0
 def _cleanup_(self):
 self.pagesize = 0
+def get(self):
+pagesize = self.pagesize
+if pagesize == 0:
+pagesize = rffi.cast(lltype.Signed, legacy_getpagesize())
+self.pagesize = pagesize
+return pagesize
+
 posixpagesize = PosixPageSize()
 
 def clear_large_memory_chunk(baseaddr, size):
 from rpython.rlib import rmmap
 
-pagesize = posixpagesize.pagesize
-if pagesize == 0:
-pagesize = rffi.cast(lltype.Signed, legacy_getpagesize())
-posixpagesize.pagesize = pagesize
-
+pagesize = posixpagesize.get()
 if size > 2 * pagesize:
 lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1)
 if lowbits: # clear the initial misaligned part, if any
@@ -435,6 +440,17 @@
 if size > 0:# clear the final misaligned part, if any
 llmemory.raw_memclear(baseaddr, size)
 
+def madvise_arena_free(baseaddr, size):
+from rpython.rlib import rmmap
+
+pagesize = posixpagesize.get()
+baseaddr = rffi.cast(lltype.Signed, baseaddr)
+aligned_addr = (baseaddr + pagesize - 1) & (pagesize - 1)
+size -= (aligned_addr - baseaddr)
+if size >= pagesize:
+rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr),
+   size & (pagesize - 1))
+
 else:
 # XXX any better implementation on Windows?
 # Should use VirtualAlloc() to reserve the range of pages,
@@ -443,6 +459,12 @@
 # them immediately.
 clear_large_memory_chunk = llmemory.raw_memclear
 
+def madvise_arena_free(baseaddr, size):
+"""XXX find a Windows equivalent?
+'baseaddr' is in the middle of memory obtained with the C malloc()...
+"""
+
+
 if os.name == "posix":
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 _eci = ExternalCompilationInfo(includes=['sys/mman.h'])
@@ -509,6 +531,8 @@
 clear_large_memory_chunk(arena_addr, size)
 elif zero == 3:
 llop.raw_memset(lltype.Void, arena_addr, ord('#'), size)
+elif zero == 4:
+madvise_arena_free(arena_addr, size)
 else:
 llmemory.raw_memclear(arena_addr, size)
 llimpl_arena_reset._always_inline_ = True
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit