Author: Ronan Lamy <ronan.l...@gmail.com> Branch: Changeset: r75984:1ac18445e487 Date: 2015-02-18 17:11 +0000 http://bitbucket.org/pypy/pypy/changeset/1ac18445e487/
Log: merge branch 'framestate2' Refactor FrameState. diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -12,8 +12,7 @@ from rpython.flowspace.argument import CallSpec from rpython.flowspace.model import (Constant, Variable, Block, Link, c_last_exception, const, FSException) -from rpython.flowspace.framestate import (FrameState, recursively_unflatten, - recursively_flatten) +from rpython.flowspace.framestate import FrameState from rpython.flowspace.specialcase import (rpython_print_item, rpython_print_newline) from rpython.flowspace.operation import op @@ -278,6 +277,7 @@ "cmp_exc_match", ] + class FlowContext(object): def __init__(self, graph, code): self.graph = graph @@ -307,112 +307,91 @@ The locals are ordered according to self.pycode.signature. """ - self.valuestackdepth = code.co_nlocals - self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals) + self.nlocals = code.co_nlocals + self.locals_w = [None] * code.co_nlocals + self.stack = [] + + @property + def stackdepth(self): + return len(self.stack) def pushvalue(self, w_object): - depth = self.valuestackdepth - self.locals_stack_w[depth] = w_object - self.valuestackdepth = depth + 1 + self.stack.append(w_object) def popvalue(self): - depth = self.valuestackdepth - 1 - assert depth >= self.pycode.co_nlocals, "pop from empty value stack" - w_object = self.locals_stack_w[depth] - self.locals_stack_w[depth] = None - self.valuestackdepth = depth - return w_object + return self.stack.pop() def peekvalue(self, index_from_top=0): # NOTE: top of the stack is peekvalue(0). - index = self.valuestackdepth + ~index_from_top - assert index >= self.pycode.co_nlocals, ( - "peek past the bottom of the stack") - return self.locals_stack_w[index] + index = ~index_from_top + return self.stack[index] def settopvalue(self, w_object, index_from_top=0): - index = self.valuestackdepth + ~index_from_top - assert index >= self.pycode.co_nlocals, ( - "settop past the bottom of the stack") - self.locals_stack_w[index] = w_object + index = ~index_from_top + self.stack[index] = w_object def popvalues(self, n): - values_w = [self.popvalue() for i in range(n)] - values_w.reverse() + if n == 0: + return [] + values_w = self.stack[-n:] + del self.stack[-n:] return values_w - def dropvalues(self, n): - finaldepth = self.valuestackdepth - n - for n in range(finaldepth, self.valuestackdepth): - self.locals_stack_w[n] = None - self.valuestackdepth = finaldepth - def dropvaluesuntil(self, finaldepth): - for n in range(finaldepth, self.valuestackdepth): - self.locals_stack_w[n] = None - self.valuestackdepth = finaldepth - - def save_locals_stack(self): - return self.locals_stack_w[:self.valuestackdepth] - - def restore_locals_stack(self, items_w): - self.locals_stack_w[:len(items_w)] = items_w - self.dropvaluesuntil(len(items_w)) + del self.stack[finaldepth:] def getstate(self, next_offset): - # getfastscope() can return real None, for undefined locals - data = self.save_locals_stack() - if self.last_exception is None: - data.append(Constant(None)) - data.append(Constant(None)) - else: - data.append(self.last_exception.w_type) - data.append(self.last_exception.w_value) - recursively_flatten(data) - return FrameState(data, self.blockstack[:], next_offset) + return FrameState(self.locals_w[:], self.stack[:], + self.last_exception, self.blockstack[:], next_offset) def setstate(self, state): """ Reset the context to the given frame state. """ - data = state.mergeable[:] - recursively_unflatten(data) - self.restore_locals_stack(data[:-2]) # Nones == undefined locals - if data[-2] == Constant(None): - assert data[-1] == Constant(None) - self.last_exception = None - else: - self.last_exception = FSException(data[-2], data[-1]) + self.locals_w = state.locals_w[:] + self.stack = state.stack[:] + self.last_exception = state.last_exception self.blockstack = state.blocklist[:] + self._normalize_raise_signals() + + def _normalize_raise_signals(self): + st = self.stack + for i in range(len(st)): + if isinstance(st[i], RaiseImplicit): + st[i] = Raise(st[i].w_exc) def guessbool(self, w_condition): if isinstance(w_condition, Constant): return w_condition.value return self.recorder.guessbool(self, w_condition) - def record(self, spaceop): + def maybe_merge(self): recorder = self.recorder if getattr(recorder, 'final_state', None) is not None: self.mergeblock(recorder.crnt_block, recorder.final_state) raise StopFlowing + + def record(self, spaceop): spaceop.offset = self.last_offset - recorder.append(spaceop) + self.recorder.append(spaceop) def do_op(self, op): + self.maybe_merge() self.record(op) self.guessexception(op.canraise) return op.result - def guessexception(self, exceptions, force=False): + def guessexception(self, exceptions): """ Catch possible exceptions implicitly. """ if not exceptions: return - if not force and not any(isinstance(block, (ExceptBlock, FinallyBlock)) - for block in self.blockstack): - # The implicit exception wouldn't be caught and would later get - # removed, so don't bother creating it. - return - self.recorder.guessexception(self, *exceptions) + # Implicit exceptions are ignored unless they are caught explicitly + if self.has_exc_handler(): + self.recorder.guessexception(self, *exceptions) + + def has_exc_handler(self): + return any(isinstance(block, (ExceptBlock, FinallyBlock)) + for block in self.blockstack) def build_flow(self): graph = self.graph @@ -430,35 +409,8 @@ while True: next_offset = self.handle_bytecode(next_offset) self.recorder.final_state = self.getstate(next_offset) - - except RaiseImplicit as e: - w_exc = e.w_exc - if isinstance(w_exc.w_type, Constant): - exc_cls = w_exc.w_type.value - else: - exc_cls = Exception - msg = "implicit %s shouldn't occur" % exc_cls.__name__ - w_type = Constant(AssertionError) - w_value = Constant(AssertionError(msg)) - link = Link([w_type, w_value], self.graph.exceptblock) - self.recorder.crnt_block.closeblock(link) - - except Raise as e: - w_exc = e.w_exc - if w_exc.w_type == const(ImportError): - msg = 'import statement always raises %s' % e - raise ImportError(msg) - link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock) - self.recorder.crnt_block.closeblock(link) - except StopFlowing: pass - - except Return as exc: - w_result = exc.w_value - link = Link([w_result], self.graph.returnblock) - self.recorder.crnt_block.closeblock(link) - except FlowingError as exc: if exc.ctx is None: exc.ctx = self @@ -476,14 +428,8 @@ if newstate is not None: break else: - newstate = currentstate.copy() - newblock = SpamBlock(newstate) - # unconditionally link the current block to the newblock - outputargs = currentstate.getoutputargs(newstate) - link = Link(outputargs, newblock) - currentblock.closeblock(link) + newblock = self.make_next_block(currentblock, currentstate) candidates.insert(0, newblock) - self.pendingblocks.append(newblock) return if newstate.matches(block.framestate): @@ -493,7 +439,7 @@ newblock = SpamBlock(newstate) varnames = self.pycode.co_varnames - for name, w_value in zip(varnames, newstate.mergeable): + for name, w_value in zip(varnames, newstate.locals_w): if isinstance(w_value, Variable): w_value.rename(name) # unconditionally link the current block to the newblock @@ -513,11 +459,21 @@ candidates.insert(0, newblock) self.pendingblocks.append(newblock) + def make_next_block(self, block, state): + newstate = state.copy() + newblock = SpamBlock(newstate) + # unconditionally link the current block to the newblock + outputargs = state.getoutputargs(newstate) + link = Link(outputargs, newblock) + block.closeblock(link) + self.pendingblocks.append(newblock) + return newblock + # hack for unrolling iterables, don't use this def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) - stack_items_w = self.locals_stack_w - for i in range(self.valuestackdepth - 1, self.pycode.co_nlocals - 1, -1): + stack_items_w = self.stack + for i in range(self.stackdepth - 1, - 1, -1): w_v = stack_items_w[i] if isinstance(w_v, Constant): if w_v.value is oldvalue: @@ -541,7 +497,7 @@ if isinstance(signal, block.handles): return block.handle(self, signal) block.cleanupstack(self) - return signal.nomoreblocks() + return signal.nomoreblocks(self) def getlocalvarname(self, index): return self.pycode.co_varnames[index] @@ -870,7 +826,7 @@ op.simple_call(w_exitfunc, w_None, w_None, w_None).eval(self) def LOAD_FAST(self, varindex): - w_value = self.locals_stack_w[varindex] + w_value = self.locals_w[varindex] if w_value is None: raise FlowingError("Local variable referenced before assignment") self.pushvalue(w_value) @@ -915,7 +871,7 @@ def STORE_FAST(self, varindex): w_newvalue = self.popvalue() assert w_newvalue is not None - self.locals_stack_w[varindex] = w_newvalue + self.locals_w[varindex] = w_newvalue if isinstance(w_newvalue, Variable): w_newvalue.rename(self.getlocalvarname(varindex)) @@ -1128,11 +1084,11 @@ op.simple_call(w_append_meth, w_value).eval(self) def DELETE_FAST(self, varindex): - if self.locals_stack_w[varindex] is None: + if self.locals_w[varindex] is None: varname = self.getlocalvarname(varindex) message = "local variable '%s' referenced before assignment" raise UnboundLocalError(message, varname) - self.locals_stack_w[varindex] = None + self.locals_w[varindex] = None def STORE_MAP(self, oparg): w_key = self.popvalue() @@ -1220,25 +1176,32 @@ WHY_CONTINUE, Continue WHY_YIELD not needed """ - def nomoreblocks(self): + def nomoreblocks(self, ctx): raise BytecodeCorruption("misplaced bytecode - should not return") + def __eq__(self, other): + return type(other) is type(self) and other.args == self.args + class Return(FlowSignal): """Signals a 'return' statement. - Argument is the wrapped object to return.""" - + Argument is the wrapped object to return. + """ def __init__(self, w_value): self.w_value = w_value - def nomoreblocks(self): - raise Return(self.w_value) + def nomoreblocks(self, ctx): + w_result = self.w_value + link = Link([w_result], ctx.graph.returnblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing - def state_unpack_variables(self): + @property + def args(self): return [self.w_value] @staticmethod - def state_pack_variables(w_value): + def rebuild(w_value): return Return(w_value) class Raise(FlowSignal): @@ -1248,28 +1211,48 @@ def __init__(self, w_exc): self.w_exc = w_exc - def nomoreblocks(self): - raise self + def nomoreblocks(self, ctx): + w_exc = self.w_exc + if w_exc.w_type == const(ImportError): + msg = 'import statement always raises %s' % self + raise ImportError(msg) + link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing - def state_unpack_variables(self): + @property + def args(self): return [self.w_exc.w_type, self.w_exc.w_value] - @staticmethod - def state_pack_variables(w_type, w_value): - return Raise(FSException(w_type, w_value)) + @classmethod + def rebuild(cls, w_type, w_value): + return cls(FSException(w_type, w_value)) class RaiseImplicit(Raise): """Signals an exception raised implicitly""" + def nomoreblocks(self, ctx): + w_exc = self.w_exc + if isinstance(w_exc.w_type, Constant): + exc_cls = w_exc.w_type.value + else: + exc_cls = Exception + msg = "implicit %s shouldn't occur" % exc_cls.__name__ + w_type = Constant(AssertionError) + w_value = Constant(AssertionError(msg)) + link = Link([w_type, w_value], ctx.graph.exceptblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing class Break(FlowSignal): """Signals a 'break' statement.""" - def state_unpack_variables(self): + @property + def args(self): return [] @staticmethod - def state_pack_variables(): + def rebuild(): return Break.singleton Break.singleton = Break() @@ -1281,11 +1264,12 @@ def __init__(self, jump_to): self.jump_to = jump_to - def state_unpack_variables(self): + @property + def args(self): return [const(self.jump_to)] @staticmethod - def state_pack_variables(w_jump_to): + def rebuild(w_jump_to): return Continue(w_jump_to.value) @@ -1295,21 +1279,21 @@ def __init__(self, ctx, handlerposition): self.handlerposition = handlerposition - self.valuestackdepth = ctx.valuestackdepth + self.stackdepth = ctx.stackdepth def __eq__(self, other): return (self.__class__ is other.__class__ and self.handlerposition == other.handlerposition and - self.valuestackdepth == other.valuestackdepth) + self.stackdepth == other.stackdepth) def __ne__(self, other): return not (self == other) def __hash__(self): - return hash((self.handlerposition, self.valuestackdepth)) + return hash((self.handlerposition, self.stackdepth)) def cleanupstack(self, ctx): - ctx.dropvaluesuntil(self.valuestackdepth) + ctx.dropvaluesuntil(self.stackdepth) def handle(self, ctx, unroller): raise NotImplementedError diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -1,21 +1,50 @@ -from rpython.flowspace.model import Variable, Constant +from rpython.flowspace.model import Variable, Constant, FSException from rpython.rlib.unroll import SpecTag +def _copy(v): + from rpython.flowspace.flowcontext import FlowSignal + if isinstance(v, Variable): + return Variable(v) + elif isinstance(v, FlowSignal): + vars = [_copy(var) for var in v.args] + return v.rebuild(*vars) + else: + return v + +def _union(seq1, seq2): + return [union(v1, v2) for v1, v2 in zip(seq1, seq2)] + class FrameState(object): - def __init__(self, mergeable, blocklist, next_offset): - self.mergeable = mergeable + def __init__(self, locals_w, stack, last_exception, blocklist, next_offset): + self.locals_w = locals_w + self.stack = stack + self.last_exception = last_exception self.blocklist = blocklist self.next_offset = next_offset + self._mergeable = None + + @property + def mergeable(self): + if self._mergeable is not None: + return self._mergeable + self._mergeable = data = self.locals_w + self.stack + if self.last_exception is None: + data.append(Constant(None)) + data.append(Constant(None)) + else: + data.append(self.last_exception.w_type) + data.append(self.last_exception.w_value) + recursively_flatten(data) + return data def copy(self): "Make a copy of this state in which all Variables are fresh." - newstate = [] - for w in self.mergeable: - if isinstance(w, Variable): - w = Variable(w) - newstate.append(w) - return FrameState(newstate, self.blocklist, self.next_offset) + exc = self.last_exception + if exc is not None: + exc = FSException(_copy(exc.w_type), _copy(exc.w_value)) + return FrameState(map(_copy, self.locals_w), map(_copy, self.stack), + exc, self.blocklist, self.next_offset) def getvariables(self): return [w for w in self.mergeable if isinstance(w, Variable)] @@ -33,18 +62,31 @@ return False return True + def _exc_args(self): + if self.last_exception is None: + return [Constant(None), Constant(None)] + else: + return [self.last_exception.w_type, + self.last_exception.w_value] + def union(self, other): """Compute a state that is at least as general as both self and other. A state 'a' is more general than a state 'b' if all Variables in 'b' are also Variables in 'a', but 'a' may have more Variables. """ - newstate = [] try: - for w1, w2 in zip(self.mergeable, other.mergeable): - newstate.append(union(w1, w2)) + locals = _union(self.locals_w, other.locals_w) + stack = _union(self.stack, other.stack) + if self.last_exception is None and other.last_exception is None: + exc = None + else: + args1 = self._exc_args() + args2 = other._exc_args() + exc = FSException(union(args1[0], args2[0]), + union(args1[1], args2[1])) except UnionError: return None - return FrameState(newstate, self.blocklist, self.next_offset) + return FrameState(locals, stack, exc, self.blocklist, self.next_offset) def getoutputargs(self, targetstate): "Return the output arguments needed to link self to targetstate." @@ -61,6 +103,7 @@ def union(w1, w2): "Union of two variables or constants." + from rpython.flowspace.flowcontext import FlowSignal if w1 == w2: return w1 if w1 is None or w2 is None: @@ -69,38 +112,21 @@ if isinstance(w1, Variable) or isinstance(w2, Variable): return Variable() # new fresh Variable if isinstance(w1, Constant) and isinstance(w2, Constant): - # FlowSignal represent stack unrollers in the stack. - # They should not be merged because they will be unwrapped. - # This is needed for try:except: and try:finally:, though - # it makes the control flow a bit larger by duplicating the - # handlers. - dont_merge_w1 = w1 in UNPICKLE_TAGS or isinstance(w1.value, SpecTag) - dont_merge_w2 = w2 in UNPICKLE_TAGS or isinstance(w2.value, SpecTag) - if dont_merge_w1 or dont_merge_w2: + if isinstance(w1.value, SpecTag) or isinstance(w2.value, SpecTag): raise UnionError else: return Variable() # generalize different constants + if isinstance(w1, FlowSignal) and isinstance(w2, FlowSignal): + if type(w1) is not type(w2): + raise UnionError + vars = [union(v1, v2) for v1, v2 in zip(w1.args, w2.args)] + return w1.rebuild(*vars) + if isinstance(w1, FlowSignal) or isinstance(w2, FlowSignal): + raise UnionError raise TypeError('union of %r and %r' % (w1.__class__.__name__, w2.__class__.__name__)) -# ____________________________________________________________ -# -# We have to flatten out the state of the frame into a list of -# Variables and Constants. This is done above by collecting the -# locals and the items on the value stack, but the latter may contain -# FlowSignal. We have to handle these specially, because -# some of them hide references to more Variables and Constants. -# The trick is to flatten ("pickle") them into the list so that the -# extra Variables show up directly in the list too. - -class PickleTag: - pass - -PICKLE_TAGS = {} -UNPICKLE_TAGS = {} - - def recursively_flatten(lst): from rpython.flowspace.flowcontext import FlowSignal i = 0 @@ -109,22 +135,4 @@ if not isinstance(unroller, FlowSignal): i += 1 else: - vars = unroller.state_unpack_variables() - key = unroller.__class__, len(vars) - try: - tag = PICKLE_TAGS[key] - except KeyError: - tag = PICKLE_TAGS[key] = Constant(PickleTag()) - UNPICKLE_TAGS[tag] = key - lst[i:i + 1] = [tag] + vars - - -def recursively_unflatten(lst): - for i in xrange(len(lst) - 1, -1, -1): - item = lst[i] - if item in UNPICKLE_TAGS: - unrollerclass, argcount = UNPICKLE_TAGS[item] - arguments = lst[i + 1:i + 1 + argcount] - del lst[i + 1:i + 1 + argcount] - unroller = unrollerclass.state_pack_variables(*arguments) - lst[i] = unroller + lst[i:i + 1] = unroller.args diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -517,7 +517,7 @@ ctx.replace_in_stack(it, next_unroller) return const(v) w_item = ctx.do_op(self) - ctx.guessexception([StopIteration, RuntimeError], force=True) + ctx.recorder.guessexception(ctx, StopIteration, RuntimeError) return w_item class GetAttr(SingleDispatchMixin, HLOperation): diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py --- a/rpython/flowspace/pygraph.py +++ b/rpython/flowspace/pygraph.py @@ -11,10 +11,10 @@ def __init__(self, func, code): from rpython.flowspace.flowcontext import SpamBlock - data = [None] * code.co_nlocals + locals = [None] * code.co_nlocals for i in range(code.formalargcount): - data[i] = Variable(code.co_varnames[i]) - state = FrameState(data + [Constant(None), Constant(None)], [], 0) + locals[i] = Variable(code.co_varnames[i]) + state = FrameState(locals, [], None, [], 0) initialblock = SpamBlock(state) super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock) self.func = func diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py new file mode 100644 --- /dev/null +++ b/rpython/flowspace/test/test_flowcontext.py @@ -0,0 +1,15 @@ +""" Unit tests for flowcontext.py """ +import pytest +from rpython.flowspace.model import Variable, FSException +from rpython.flowspace.flowcontext import ( + Return, Raise, RaiseImplicit, Continue, Break) + +@pytest.mark.parametrize('signal', [ + Return(Variable()), + Raise(FSException(Variable(), Variable())), + RaiseImplicit(FSException(Variable(), Variable())), + Break(), + Continue(42), +]) +def test_signals(signal): + assert signal.rebuild(*signal.args) == signal diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py --- a/rpython/flowspace/test/test_framestate.py +++ b/rpython/flowspace/test/test_framestate.py @@ -15,7 +15,7 @@ ctx = FlowContext(graph, code) # hack the frame ctx.setstate(graph.startblock.framestate) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(None) + ctx.locals_w[-1] = Constant(None) return ctx def func_simple(x): @@ -31,7 +31,7 @@ def test_neq_hacked_framestate(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) assert not fs1.matches(fs2) @@ -44,7 +44,7 @@ def test_union_on_hacked_framestates(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) assert fs1.union(fs2).matches(fs2) # fs2 is more general assert fs2.union(fs1).matches(fs2) # fs2 is more general @@ -52,7 +52,7 @@ def test_restore_frame(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() ctx.setstate(fs1) assert fs1.matches(ctx.getstate(0)) @@ -71,26 +71,25 @@ def test_getoutputargs(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) outputargs = fs1.getoutputargs(fs2) # 'x' -> 'x' is a Variable # locals_w[n-1] -> locals_w[n-1] is Constant(None) - assert outputargs == [ctx.locals_stack_w[0], Constant(None)] + assert outputargs == [ctx.locals_w[0], Constant(None)] def test_union_different_constants(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(42) + ctx.locals_w[-1] = Constant(42) fs2 = ctx.getstate(0) fs3 = fs1.union(fs2) ctx.setstate(fs3) - assert isinstance(ctx.locals_stack_w[ctx.pycode.co_nlocals-1], - Variable) # generalized + assert isinstance(ctx.locals_w[-1], Variable) # generalized def test_union_spectag(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(SpecTag()) + ctx.locals_w[-1] = Constant(SpecTag()) fs2 = ctx.getstate(0) assert fs1.union(fs2) is None # UnionError _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit