Author: Antonio Cuni <anto.c...@gmail.com> Branch: py3k Changeset: r54972:cf14a5197515 Date: 2012-05-08 14:46 +0200 http://bitbucket.org/pypy/pypy/changeset/cf14a5197515/
Log: hg merge default diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -51,8 +51,15 @@ `Download`_ a binary or install from `source`_. Some Linux and Mac systems may have ROOT provided in the list of scientific software of their packager. -A current, standalone version of Reflex should be provided at some point, -once the dependencies and general packaging have been thought out. +If, however, you prefer a standalone version of Reflex, the best is to get +this `recent snapshot`_, and install like so:: + + $ tar jxf reflex-2012-05-02.tar.bz2 + $ cd reflex-2012-05-02 + $ build/autogen + $ ./configure <usual set of options such as --prefix> + $ make && make install + Also, make sure you have a version of `gccxml`_ installed, which is most easily provided by the packager of your system. If you read up on gccxml, you'll probably notice that it is no longer being @@ -61,12 +68,13 @@ .. _`Download`: http://root.cern.ch/drupal/content/downloading-root .. _`source`: http://root.cern.ch/drupal/content/installing-root-source +.. _`recent snapshot`: http://cern.ch/wlav/reflex-2012-05-02.tar.bz2 .. _`gccxml`: http://www.gccxml.org Next, get the `PyPy sources`_, select the reflex-support branch, and build pypy-c. For the build to succeed, the ``$ROOTSYS`` environment variable must point to -the location of your ROOT installation:: +the location of your ROOT (or standalone Reflex) installation:: $ hg clone https://bitbucket.org/pypy/pypy $ cd pypy diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -138,12 +138,8 @@ useful: they will appear to stay alive for a bit longer in PyPy, and suddenly they will really be dead, raising a ``ReferenceError`` on the next access. Any code that uses weak proxies must carefully catch such -``ReferenceError`` at any place that uses them. - -As a side effect, the ``finally`` clause inside a generator will be executed -only when the generator object is garbage collected (see `issue 736`__). - -.. __: http://bugs.pypy.org/issue736 +``ReferenceError`` at any place that uses them. (Or, better yet, don't use +``weakref.proxy()`` at all; use ``weakref.ref()``.) There are a few extra implications for the difference in the GC. Most notably, if an object has a ``__del__``, the ``__del__`` is never called more @@ -162,7 +158,10 @@ example, a generator left pending in the middle is --- again --- garbage-collected later in PyPy than in CPython. You can see the difference if the ``yield`` keyword it is suspended at is itself -enclosed in a ``try:`` or a ``with:`` block. +enclosed in a ``try:`` or a ``with:`` block. This shows up for example +as `issue 736`__. + +.. __: http://bugs.pypy.org/issue736 Using the default GC called ``minimark``, the built-in function ``id()`` works like it does in CPython. With other GCs it returns numbers that @@ -186,7 +185,8 @@ Even more obscure: the same is true, for old-style classes, if you attach the ``__del__`` to an instance (even in CPython this does not work with new-style classes). You get a RuntimeWarning in PyPy. To fix these cases -just make sure there is a ``__del__`` method in the class to start with. +just make sure there is a ``__del__`` method in the class to start with +(even containing only ``pass``; replacing or overriding it later works fine). Subclasses of built-in types diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1633,8 +1633,6 @@ 'UnicodeTranslateError', 'ValueError', 'ZeroDivisionError', - 'UnicodeEncodeError', - 'UnicodeDecodeError', ] if sys.platform.startswith("win"): diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1272,10 +1272,6 @@ w(self.valuestackdepth)]) def handle(self, frame, unroller): - next_instr = self.really_handle(frame, unroller) # JIT hack - return r_uint(next_instr) - - def really_handle(self, frame, unroller): """ Purely abstract method """ raise NotImplementedError @@ -1287,17 +1283,17 @@ _opname = 'SETUP_LOOP' handling_mask = SBreakLoop.kind | SContinueLoop.kind - def really_handle(self, frame, unroller): + def handle(self, frame, unroller): if isinstance(unroller, SContinueLoop): # re-push the loop block without cleaning up the value stack, # and jump to the beginning of the loop, stored in the # exception's argument frame.append_block(self) - return unroller.jump_to + return r_uint(unroller.jump_to) else: # jump to the end of the loop self.cleanupstack(frame) - return self.handlerposition + return r_uint(self.handlerposition) class ExceptBlock(FrameBlock): @@ -1307,7 +1303,7 @@ _opname = 'SETUP_EXCEPT' handling_mask = SApplicationException.kind - def really_handle(self, frame, unroller): + def handle(self, frame, unroller): # push the exception to the value stack for inspection by the # exception handler (the code after the except:) self.cleanupstack(frame) @@ -1327,7 +1323,7 @@ frame.pushvalue(operationerr.get_w_value(frame.space)) frame.pushvalue(operationerr.w_type) frame.last_exception = operationerr - return self.handlerposition # jump to the handler + return r_uint(self.handlerposition) # jump to the handler class FinallyBlock(FrameBlock): @@ -1349,7 +1345,7 @@ frame.pushvalue(frame.space.w_None) frame.pushvalue(frame.space.w_None) - def really_handle(self, frame, unroller): + def handle(self, frame, unroller): # any abnormal reason for unrolling a finally: triggers the end of # the block unrolling and the entering the finally: handler. # see comments in cleanup(). @@ -1364,9 +1360,8 @@ frame.pushvalue(frame.space.w_None) if operationerr and self.restore_last_exception: frame.last_exception = operationerr - return self.handlerposition # jump to the handler + return r_uint(self.handlerposition) # jump to the handler - class WithBlock(FinallyBlock): @@ -1374,11 +1369,11 @@ _immutable_ = True restore_last_exception = False - def really_handle(self, frame, unroller): + def handle(self, frame, unroller): if (frame.space.full_exceptions and isinstance(unroller, SApplicationException)): unroller.operr.normalize_exception(frame.space) - return FinallyBlock.really_handle(self, frame, unroller) + return FinallyBlock.handle(self, frame, unroller) block_classes = {'SETUP_LOOP': LoopBlock, 'SETUP_EXCEPT': ExceptBlock, diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py --- a/pypy/jit/metainterp/optimizeopt/earlyforce.py +++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py @@ -7,7 +7,8 @@ opnum = op.getopnum() if (opnum != rop.SETFIELD_GC and opnum != rop.SETARRAYITEM_GC and - opnum != rop.QUASIIMMUT_FIELD): + opnum != rop.QUASIIMMUT_FIELD and + opnum != rop.SAME_AS): for arg in op.getarglist(): if arg in self.optimizer.values: diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py --- a/pypy/jit/metainterp/optimizeopt/optimizer.py +++ b/pypy/jit/metainterp/optimizeopt/optimizer.py @@ -652,8 +652,15 @@ arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint()) self.optimize_default(op) + # These are typically removed already by OptRewrite, but it can be + # dissabled and unrolling emits some SAME_AS ops to setup the + # optimizier state. These needs to always be optimized out. + def optimize_SAME_AS(self, op): + self.make_equal_to(op.result, self.getvalue(op.getarg(0))) - + def optimize_MARK_OPAQUE_PTR(self, op): + value = self.getvalue(op.getarg(0)) + self.optimizer.opaque_pointers[value] = True dispatch_opt = make_dispatcher_method(Optimizer, 'optimize_', default=Optimizer.optimize_default) diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py --- a/pypy/jit/metainterp/optimizeopt/unroll.py +++ b/pypy/jit/metainterp/optimizeopt/unroll.py @@ -335,9 +335,13 @@ args[short_inputargs[i]] = jmp_to_short_args[i] self.short_inliner = Inliner(short_inputargs, jmp_to_short_args) - for op in self.short[1:]: + i = 1 + while i < len(self.short): + # Note that self.short might be extended during this loop + op = self.short[i] newop = self.short_inliner.inline_op(op) self.optimizer.send_extra_operation(newop) + i += 1 # Import boxes produced in the preamble but used in the loop newoperations = self.optimizer.get_newoperations() diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -107,7 +107,7 @@ obj.double_member = 9.25; assert obj.double_member == 9.25 obj.longlong_member = -2**59; assert obj.longlong_member == -2**59 obj.ulonglong_member = 2**63; assert obj.ulonglong_member == 2**63 - obj.ssizet_member = 2**31; assert obj.ssizet_member == 2**31 + obj.ssizet_member = sys.maxint;assert obj.ssizet_member == sys.maxint # def test_staticmethod(self): diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -6,6 +6,7 @@ import re from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root +from pypy.interpreter.error import OperationError from pypy.module.micronumpy import interp_boxes from pypy.module.micronumpy.interp_dtype import get_dtype_cache from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray, @@ -39,11 +40,11 @@ THREE_ARG_FUNCTIONS = ['where'] class FakeSpace(object): - w_ValueError = None - w_TypeError = None - w_IndexError = None - w_OverflowError = None - w_NotImplementedError = None + w_ValueError = "ValueError" + w_TypeError = "TypeError" + w_IndexError = "IndexError" + w_OverflowError = "OverflowError" + w_NotImplementedError = "NotImplementedError" w_None = None w_bool = "bool" @@ -126,8 +127,13 @@ return w_obj.intval elif isinstance(w_obj, FloatObject): return int(w_obj.floatval) + elif isinstance(w_obj, SliceObject): + raise OperationError(self.w_TypeError, self.wrap("slice.")) raise NotImplementedError + def index(self, w_obj): + return self.wrap(self.int_w(w_obj)) + def str_w(self, w_obj): if isinstance(w_obj, StringObject): return w_obj.v diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -350,12 +350,31 @@ if shape_len == 1: if space.isinstance_w(w_idx, space.w_int): return True + + try: + value = space.int_w(space.index(w_idx)) + return True + except OperationError: + pass + + try: + value = space.int_w(w_idx) + return True + except OperationError: + pass + if space.isinstance_w(w_idx, space.w_slice): return False elif (space.isinstance_w(w_idx, space.w_slice) or space.isinstance_w(w_idx, space.w_int)): return False - lgt = space.len_w(w_idx) + + try: + lgt = space.len_w(w_idx) + except OperationError: + raise OperationError(space.w_IndexError, + space.wrap("index must be either an int or a sequence.")) + if lgt > shape_len: raise OperationError(space.w_IndexError, space.wrap("invalid index")) @@ -1030,8 +1049,21 @@ @jit.unroll_safe def _index_of_single_item(self, space, w_idx): - if space.isinstance_w(w_idx, space.w_int): - idx = space.int_w(w_idx) + is_valid = False + try: + idx = space.int_w(space.index(w_idx)) + is_valid = True + except OperationError: + pass + + if not is_valid: + try: + idx = space.int_w(w_idx) + is_valid = True + except OperationError: + pass + + if is_valid: if idx < 0: idx = self.shape[0] + idx if idx < 0 or idx >= self.shape[0]: diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py --- a/pypy/module/micronumpy/test/test_base.py +++ b/pypy/module/micronumpy/test/test_base.py @@ -11,6 +11,7 @@ import sys class BaseNumpyAppTest(object): + @classmethod def setup_class(cls): py.test.py3k_skip("micronumpy not supported on py3k") if option.runappdirect: diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -8,7 +8,6 @@ from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest - class MockDtype(object): class itemtype(object): @staticmethod @@ -195,6 +194,36 @@ assert _to_coords(13, 'F') == [1, 0, 2] class AppTestNumArray(BaseNumpyAppTest): + def w_CustomIndexObject(self, index): + class CustomIndexObject(object): + def __init__(self, index): + self.index = index + def __index__(self): + return self.index + + return CustomIndexObject(index) + + def w_CustomIndexIntObject(self, index, value): + class CustomIndexIntObject(object): + def __init__(self, index, value): + self.index = index + self.value = value + def __index__(self): + return self.index + def __int__(self): + return self.value + + return CustomIndexIntObject(index, value) + + def w_CustomIntObject(self, value): + class CustomIntObject(object): + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + return CustomIntObject(value) + def test_ndarray(self): from _numpypy import ndarray, array, dtype @@ -329,6 +358,28 @@ assert a[1, 3] == 8 assert a.T[1, 2] == 11 + def test_getitem_obj_index(self): + from _numpypy import arange + + a = arange(10) + + assert a[self.CustomIndexObject(1)] == 1 + + def test_getitem_obj_prefer_index_to_int(self): + from _numpypy import arange + + a = arange(10) + + + assert a[self.CustomIndexIntObject(0, 1)] == 0 + + def test_getitem_obj_int(self): + from _numpypy import arange + + a = arange(10) + + assert a[self.CustomIntObject(1)] == 1 + def test_setitem(self): from _numpypy import array a = array(range(5)) @@ -348,6 +399,48 @@ for i in xrange(5): assert a[i] == i + def test_setitem_obj_index(self): + from _numpypy import arange + + a = arange(10) + + a[self.CustomIndexObject(1)] = 100 + assert a[1] == 100 + + def test_setitem_obj_prefer_index_to_int(self): + from _numpypy import arange + + a = arange(10) + + a[self.CustomIndexIntObject(0, 1)] = 100 + assert a[0] == 100 + + def test_setitem_obj_int(self): + from _numpypy import arange + + a = arange(10) + + a[self.CustomIntObject(1)] = 100 + + assert a[1] == 100 + + def test_access_swallow_exception(self): + class ErrorIndex(object): + def __index__(self): + return 1 / 0 + + class ErrorInt(object): + def __int__(self): + return 1 / 0 + + # numpy will swallow errors in __int__ and __index__ and + # just raise IndexError. + + from _numpypy import arange + a = arange(10) + raises(IndexError, "a[ErrorIndex()] == 0") + raises(IndexError, "a[ErrorInt()] == 0") + def test_setslice_array(self): from _numpypy import array a = array(range(5)) diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py --- a/pypy/module/thread/__init__.py +++ b/pypy/module/thread/__init__.py @@ -21,7 +21,7 @@ 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.Lock', 'RLock': 'os_lock.W_RLock', - '_local': 'os_local.Local', + #'_local': 'os_local.Local', 'TIMEOUT_MAX': 'space.wrap(float(os_lock.TIMEOUT_MAX) / 1000000.0)', 'error': 'space.fromcache(error.Cache).w_error', } @@ -39,3 +39,7 @@ from pypy.module.thread.os_thread import reinit_threads add_fork_hook('child', reinit_threads) + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + if self.space.config.translation.rweakref: + self.extra_interpdef('_local', 'os_local.Local') diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -5,7 +5,7 @@ # This module adds a global lock to an object space. # If multiple threads try to execute simultaneously in this space, # all but one will be blocked. The other threads get a chance to run -# from time to time, using the hook yield_thread(). +# from time to time, using the periodic action GILReleaseAction. from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error @@ -30,7 +30,6 @@ if not thread.gil_allocate(): raise wrap_thread_error(space, "can't allocate GIL") self.gil_ready = True - self.enter_thread(space) # setup the main thread result = True else: result = False # already set up @@ -51,8 +50,6 @@ self.gil_ready = False self.setup_threads(space) - def yield_thread(self): - do_yield_thread() class GILReleaseAction(PeriodicAsyncAction): """An action called every sys.checkinterval bytecodes. It releases diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -1,5 +1,6 @@ -from pypy.module.thread import ll_thread as thread -from pypy.interpreter.baseobjspace import Wrappable +from pypy.rlib.rweakref import RWeakKeyDictionary +from pypy.interpreter.baseobjspace import Wrappable, W_Root +from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty, descr_get_dict) @@ -9,16 +10,23 @@ def __init__(self, space, initargs): self.initargs = initargs - ident = thread.get_ident() - self.dicts = {ident: space.newdict(instance=True)} + self.dicts = RWeakKeyDictionary(ExecutionContext, W_Root) + # The app-level __init__() will be called by the general + # instance-creation logic. It causes getdict() to be + # immediately called. If we don't prepare and set a w_dict + # for the current thread, then this would in cause getdict() + # to call __init__() a second time. + ec = space.getexecutioncontext() + w_dict = space.newdict(instance=True) + self.dicts.set(ec, w_dict) def getdict(self, space): - ident = thread.get_ident() - try: - w_dict = self.dicts[ident] - except KeyError: + ec = space.getexecutioncontext() + w_dict = self.dicts.get(ec) + if w_dict is None: # create a new dict for this thread - w_dict = self.dicts[ident] = space.newdict(instance=True) + w_dict = space.newdict(instance=True) + self.dicts.set(ec, w_dict) # call __init__ try: w_self = space.wrap(self) @@ -27,10 +35,9 @@ space.call_obj_args(w_init, w_self, self.initargs) except: # failed, forget w_dict and propagate the exception - del self.dicts[ident] + self.dicts.set(ec, None) raise # ready - space.threadlocals.atthreadexit(space, finish_thread, self) return w_dict def descr_local__new__(space, w_subtype, __args__): @@ -48,8 +55,3 @@ __init__ = interp2app(Local.descr_local__init__), __dict__ = GetSetProperty(descr_get_dict, cls=Local), ) - -def finish_thread(w_obj): - assert isinstance(w_obj, Local) - ident = thread.get_ident() - del w_obj.dicts[ident] diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -90,7 +90,6 @@ bootstrapper.nbthreads += 1 bootstrapper.release() # run! - space.threadlocals.enter_thread(space) try: bootstrapper.run(space, w_callable, args) finally: diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -55,7 +55,7 @@ assert state.datalen3 == len(state.data) assert state.datalen4 == len(state.data) debug_print(main, i, state.datalen4) - state.threadlocals.yield_thread() + gil.do_yield_thread() assert i == j j += 1 def bootstrap(): diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -48,7 +48,7 @@ def test_local_init(self): import _thread - tags = [1, 2, 3, 4, 5, 54321] + tags = ['???', 1, 2, 3, 4, 5, 54321] seen = [] raises(TypeError, _thread._local, a=1) @@ -61,6 +61,7 @@ x = X(42) assert x.tag == 54321 + assert x.tag == 54321 def f(): seen.append(x.tag) for i in range(5): @@ -69,7 +70,7 @@ seen1 = seen[:] seen1.sort() assert seen1 == [1, 2, 3, 4, 5] - assert tags == [] + assert tags == ['???'] def test_local_setdict(self): import _thread @@ -87,3 +88,23 @@ _thread.start_new_thread(f, (i,)) self.waitfor(lambda: len(done) == 5, delay=2) assert len(done) == 5 + + def test_local_is_not_immortal(self): + import thread, gc, time + class Local(thread._local): + def __del__(self): + done.append('del') + done = [] + def f(): + assert not hasattr(l, 'foo') + l.bar = 42 + done.append('ok') + self.waitfor(lambda: len(done) == 3, delay=8) + l = Local() + l.foo = 42 + thread.start_new_thread(f, ()) + self.waitfor(lambda: len(done) == 1, delay=2) + l = None + gc.collect() + assert done == ['ok', 'del'] + done.append('shutdown') diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -1,5 +1,6 @@ from pypy.module.thread import ll_thread as thread + class OSThreadLocals: """Thread-local storage for OS-level threads. For memory management, this version depends on explicit notification when @@ -51,21 +52,6 @@ def getallvalues(self): return self._valuedict - def enter_thread(self, space): - "Notification that the current thread is just starting." - ec = space.getexecutioncontext() - ec.thread_exit_funcs = [] - def leave_thread(self, space): "Notification that the current thread is about to stop." - try: - ec = space.getexecutioncontext() - while ec.thread_exit_funcs: - exit_func, w_obj = ec.thread_exit_funcs.pop() - exit_func(w_obj) - finally: - self.setvalue(None) - - def atthreadexit(self, space, exit_func, w_obj): - ec = space.getexecutioncontext() - ec.thread_exit_funcs.append((exit_func, w_obj)) + self.setvalue(None) diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py --- a/pypy/rlib/runicode.py +++ b/pypy/rlib/runicode.py @@ -510,8 +510,9 @@ pos = 0 if byteorder == 'native': if size >= 4: - bom = ((ord(s[iorder[3]]) << 24) | (ord(s[iorder[2]]) << 16) | - (ord(s[iorder[1]]) << 8) | ord(s[iorder[0]])) + bom = intmask( + (ord(s[iorder[3]]) << 24) | (ord(s[iorder[2]]) << 16) | + (ord(s[iorder[1]]) << 8) | ord(s[iorder[0]])) if BYTEORDER == 'little': if bom == BOM32_DIRECT: pos += 4 _______________________________________________ pypy-commit mailing list pypy-commit@python.org http://mail.python.org/mailman/listinfo/pypy-commit
