Author: Hakan Ardo <ha...@debian.org>
Branch: jit-improve-nested-loops
Changeset: r50876:f75c6c5a133a
Date: 2011-12-26 13:49 +0100
http://bitbucket.org/pypy/pypy/changeset/f75c6c5a133a/
Log: hg merge default
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -180,7 +180,12 @@
if name is None:
name = pyobj.func_name
if signature is None:
- signature = cpython_code_signature(pyobj.func_code)
+ if hasattr(pyobj, '_generator_next_method_of_'):
+ from pypy.interpreter.argument import Signature
+ signature = Signature(['entry']) # haaaaaack
+ defaults = ()
+ else:
+ signature = cpython_code_signature(pyobj.func_code)
if defaults is None:
defaults = pyobj.func_defaults
self.name = name
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -98,7 +98,6 @@
"Abstract. Get the expected number of locals."
raise TypeError, "abstract"
- @jit.dont_look_inside
def fast2locals(self):
# Copy values from the fastlocals to self.w_locals
if self.w_locals is None:
@@ -112,7 +111,6 @@
w_name = self.space.wrap(name)
self.space.setitem(self.w_locals, w_name, w_value)
- @jit.dont_look_inside
def locals2fast(self):
# Copy values from self.w_locals to the fastlocals
assert self.w_locals is not None
diff --git a/pypy/jit/backend/x86/assembler.py
b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -39,6 +39,7 @@
from pypy.jit.codewriter.effectinfo import EffectInfo
from pypy.jit.codewriter import longlong
from pypy.rlib.rarithmetic import intmask
+from pypy.rlib.objectmodel import compute_unique_id
# darwin requires the stack to be 16 bytes aligned on calls. Same for gcc
4.5.0,
# better safe than sorry
@@ -147,12 +148,13 @@
def finish_once(self):
if self._debug:
debug_start('jit-backend-counts')
- for struct in self.loop_run_counters:
- if struct.bridge:
- prefix = 'bridge '
+ for i in range(len(self.loop_run_counters)):
+ struct = self.loop_run_counters[i]
+ if not struct.bridge:
+ prefix = 'TargetToken(%d)' % struct.number
else:
- prefix = 'loop '
- debug_print(prefix + str(struct.number) + ':' + str(struct.i))
+ prefix = 'bridge ' + str(struct.number)
+ debug_print(prefix + ':' + str(struct.i))
debug_stop('jit-backend-counts')
def _build_float_constants(self):
@@ -422,8 +424,8 @@
self.setup(looptoken)
if log:
- self._register_counter(False, looptoken.number)
- operations = self._inject_debugging_code(looptoken, operations)
+ operations = self._inject_debugging_code(looptoken, operations,
+ False, looptoken.number)
regalloc = RegAlloc(self, self.cpu.translate_support_code)
#
@@ -489,8 +491,8 @@
self.setup(original_loop_token)
if log:
- self._register_counter(True, descr_number)
- operations = self._inject_debugging_code(faildescr, operations)
+ operations = self._inject_debugging_code(faildescr, operations,
+ True, descr_number)
arglocs = self.rebuild_faillocs_from_descr(failure_recovery)
if not we_are_translated():
@@ -597,17 +599,21 @@
return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
self.cpu.gc_ll_descr.gcrootmap)
- def _register_counter(self, bridge, number):
- if self._debug:
- # YYY very minor leak -- we need the counters to stay alive
- # forever, just because we want to report them at the end
- # of the process
- struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
- track_allocation=False)
- struct.i = 0
- struct.bridge = int(bridge)
+ def _register_counter(self, bridge, number, token):
+ # YYY very minor leak -- we need the counters to stay alive
+ # forever, just because we want to report them at the end
+ # of the process
+ struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
+ track_allocation=False)
+ struct.i = 0
+ struct.bridge = int(bridge)
+ if bridge:
struct.number = number
- self.loop_run_counters.append(struct)
+ else:
+ assert token
+ struct.number = compute_unique_id(token)
+ self.loop_run_counters.append(struct)
+ return struct
def _find_failure_recovery_bytecode(self, faildescr):
adr_jump_offset = faildescr._x86_adr_jump_offset
@@ -651,27 +657,37 @@
targettoken._x86_loop_code += rawstart
self.target_tokens_currently_compiling = None
+ def _append_debugging_code(self, operations, bridge, number, token):
+ counter = self._register_counter(bridge, number, token)
+ c_adr = ConstInt(rffi.cast(lltype.Signed, counter))
+ box = BoxInt()
+ box2 = BoxInt()
+ ops = [ResOperation(rop.GETFIELD_RAW, [c_adr],
+ box, descr=self.debug_counter_descr),
+ ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2),
+ ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
+ None, descr=self.debug_counter_descr)]
+ operations.extend(ops)
+
@specialize.argtype(1)
- def _inject_debugging_code(self, looptoken, operations):
+ def _inject_debugging_code(self, looptoken, operations, bridge, number):
if self._debug:
# before doing anything, let's increase a counter
s = 0
for op in operations:
s += op.getopnum()
looptoken._x86_debug_checksum = s
- c_adr = ConstInt(rffi.cast(lltype.Signed,
- self.loop_run_counters[-1]))
- box = BoxInt()
- box2 = BoxInt()
- ops = [ResOperation(rop.GETFIELD_RAW, [c_adr],
- box, descr=self.debug_counter_descr),
- ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2),
- ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
- None, descr=self.debug_counter_descr)]
- if operations[0].getopnum() == rop.LABEL:
- operations = [operations[0]] + ops + operations[1:]
- else:
- operations = ops + operations
+
+ newoperations = []
+ if bridge:
+ self._append_debugging_code(newoperations, bridge, number,
+ None)
+ for op in operations:
+ newoperations.append(op)
+ if op.getopnum() == rop.LABEL:
+ self._append_debugging_code(newoperations, bridge, number,
+ op.getdescr())
+ operations = newoperations
return operations
def _assemble(self, regalloc, operations):
diff --git a/pypy/jit/backend/x86/test/test_runner.py
b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -519,16 +519,23 @@
from pypy.tool.logparser import parse_log_file, extract_category
from pypy.rlib import debug
+ targettoken, preambletoken = TargetToken(), TargetToken()
loop = """
[i0]
- label(i0, descr=targettoken)
+ label(i0, descr=preambletoken)
debug_merge_point('xyz', 0)
i1 = int_add(i0, 1)
i2 = int_ge(i1, 10)
guard_false(i2) []
- jump(i1, descr=targettoken)
+ label(i1, descr=targettoken)
+ debug_merge_point('xyz', 0)
+ i11 = int_add(i1, 1)
+ i12 = int_ge(i11, 10)
+ guard_false(i12) []
+ jump(i11, descr=targettoken)
"""
- ops = parse(loop, namespace={'targettoken': TargetToken()})
+ ops = parse(loop, namespace={'targettoken': targettoken,
+ 'preambletoken': preambletoken})
debug._log = dlog = debug.DebugLog()
try:
self.cpu.assembler.set_debug(True)
@@ -537,11 +544,15 @@
self.cpu.execute_token(looptoken, 0)
# check debugging info
struct = self.cpu.assembler.loop_run_counters[0]
- assert struct.i == 10
+ assert struct.i == 1
+ struct = self.cpu.assembler.loop_run_counters[1]
+ assert struct.i == 9
self.cpu.finish_once()
finally:
debug._log = None
- assert ('jit-backend-counts', [('debug_print', 'loop -1:10')]) in dlog
+ l1 = ('debug_print', preambletoken.repr_of_descr() + ':1')
+ l2 = ('debug_print', targettoken.repr_of_descr() + ':9')
+ assert ('jit-backend-counts', [l1, l2]) in dlog
def test_debugger_checksum(self):
loop = """
diff --git a/pypy/jit/backend/x86/test/test_zrpy_platform.py
b/pypy/jit/backend/x86/test/test_zrpy_platform.py
--- a/pypy/jit/backend/x86/test/test_zrpy_platform.py
+++ b/pypy/jit/backend/x86/test/test_zrpy_platform.py
@@ -74,8 +74,8 @@
myjitdriver = jit.JitDriver(greens = [], reds = ['n'])
def entrypoint(argv):
- myjitdriver.set_param('threshold', 2)
- myjitdriver.set_param('trace_eagerness', 0)
+ jit.set_param(myjitdriver, 'threshold', 2)
+ jit.set_param(myjitdriver, 'trace_eagerness', 0)
n = 16
while n > 0:
myjitdriver.can_enter_jit(n=n)
diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py
--- a/pypy/jit/codewriter/call.py
+++ b/pypy/jit/codewriter/call.py
@@ -42,8 +42,7 @@
except AttributeError:
pass
- def is_candidate(graph):
- return policy.look_inside_graph(graph)
+ is_candidate = policy.look_inside_graph
assert len(self.jitdrivers_sd) > 0
todo = [jd.portal_graph for jd in self.jitdrivers_sd]
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -1007,25 +1007,6 @@
# a jump back to itself and possibly a few bridges ending with finnish.
# Only the operations within the loop formed by that single jump will
# be counted.
-
- # XXX hacked version, ignore and remove me when jit-targets is merged.
- loops = self.get_all_loops()
- loops = [loop for loop in loops if 'Preamble' not in repr(loop)] #XXX
- assert len(loops) == 1
- loop, = loops
- jumpop = loop.operations[-1]
- assert jumpop.getopnum() == rop.JUMP
- insns = {}
- for op in loop.operations:
- opname = op.getopname()
- insns[opname] = insns.get(opname, 0) + 1
- return self._check_insns(insns, expected, check)
-
- def check_simple_loop(self, expected=None, **check):
- # Usefull in the simplest case when we have only one trace ending with
- # a jump back to itself and possibly a few bridges ending with finnish.
- # Only the operations within the loop formed by that single jump will
- # be counted.
loops = self.get_all_loops()
assert len(loops) == 1
loop = loops[0]
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -1,10 +1,13 @@
from __future__ import with_statement
from pypy.jit.metainterp.optimizeopt.test.test_util import (
- LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot)
+ LLtypeMixin, BaseTest, Storage, _sortboxes, FakeDescrWithSnapshot,
+ FakeMetaInterpStaticData)
from pypy.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken
from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
from pypy.jit.metainterp.optimize import InvalidLoop
from py.test import raises
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method
class BaseTestMultiLabel(BaseTest):
enable_opts =
"intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
@@ -84,6 +87,8 @@
return optimized
+class OptimizeoptTestMultiLabel(BaseTestMultiLabel):
+
def test_simple(self):
ops = """
[i1]
@@ -381,6 +386,55 @@
"""
self.optimize_loop(ops, expected)
-class TestLLtype(BaseTestMultiLabel, LLtypeMixin):
+
+class OptRenameStrlen(Optimization):
+ def propagate_forward(self, op):
+ dispatch_opt(self, op)
+
+ def optimize_STRLEN(self, op):
+ newop = op.clone()
+ newop.result = op.result.clonebox()
+ self.emit_operation(newop)
+ self.make_equal_to(op.result, self.getvalue(newop.result))
+
+dispatch_opt = make_dispatcher_method(OptRenameStrlen, 'optimize_',
+ default=OptRenameStrlen.emit_operation)
+
+class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
+
+ def _do_optimize_loop(self, loop, call_pure_results):
+ from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll
+ from pypy.jit.metainterp.optimizeopt.util import args_dict
+ from pypy.jit.metainterp.optimizeopt.pure import OptPure
+
+ self.loop = loop
+ loop.call_pure_results = args_dict()
+ metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+ optimize_unroll(metainterp_sd, loop, [OptRenameStrlen(), OptPure()],
True)
+
+ def test_optimizer_renaming_boxes(self):
+ ops = """
+ [p1]
+ i1 = strlen(p1)
+ label(p1)
+ i2 = strlen(p1)
+ i3 = int_add(i2, 7)
+ jump(p1)
+ """
+ expected = """
+ [p1]
+ i1 = strlen(p1)
+ label(p1, i1)
+ i11 = same_as(i1)
+ i2 = int_add(i11, 7)
+ jump(p1, i11)
+ """
+ self.optimize_loop(ops, expected)
+
+
+
+class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
pass
+class TestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes,
LLtypeMixin):
+ pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7759,7 +7759,7 @@
jump(i0, p0, i2)
"""
self.optimize_loop(ops, expected)
-
+
class TestLLtype(OptimizeOptTest, LLtypeMixin):
pass
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py
b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -265,7 +265,12 @@
self.optimizer.importable_values[value] = imp
newvalue = self.optimizer.getvalue(op.result)
newresult = newvalue.get_key_box()
- assert newresult is op.result or newvalue.is_constant()
+ # note that emitting here SAME_AS should not happen, but
+ # in case it does, we would prefer to be suboptimal in asm
+ # to a fatal RPython exception.
+ if newresult is not op.result and not newvalue.is_constant():
+ op = ResOperation(rop.SAME_AS, [op.result], newresult)
+ self.optimizer._newoperations.append(op)
self.optimizer.flush()
self.optimizer.emitting_dissabled = False
diff --git a/pypy/module/cpyext/methodobject.py
b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -58,6 +58,7 @@
class W_PyCFunctionObject(Wrappable):
def __init__(self, space, ml, w_self, w_module=None):
self.ml = ml
+ self.name = rffi.charp2str(self.ml.c_ml_name)
self.w_self = w_self
self.w_module = w_module
@@ -69,7 +70,7 @@
flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST)
if space.is_true(w_kw) and not flags & METH_KEYWORDS:
raise OperationError(space.w_TypeError, space.wrap(
- rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword
arguments"))
+ self.name + "() takes no keyword arguments"))
func = rffi.cast(PyCFunction, self.ml.c_ml_meth)
length = space.int_w(space.len(w_args))
@@ -80,13 +81,12 @@
if length == 0:
return generic_cpy_call(space, func, w_self, None)
raise OperationError(space.w_TypeError, space.wrap(
- rffi.charp2str(self.ml.c_ml_name) + "() takes no arguments"))
+ self.name + "() takes no arguments"))
elif flags & METH_O:
if length != 1:
raise OperationError(space.w_TypeError,
space.wrap("%s() takes exactly one argument (%d
given)" % (
- rffi.charp2str(self.ml.c_ml_name),
- length)))
+ self.name, length)))
w_arg = space.getitem(w_args, space.wrap(0))
return generic_cpy_call(space, func, w_self, w_arg)
elif flags & METH_VARARGS:
@@ -199,6 +199,7 @@
__call__ = interp2app(cfunction_descr_call),
__doc__ = GetSetProperty(W_PyCFunctionObject.get_doc),
__module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject),
+ __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject),
)
W_PyCFunctionObject.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/cpyext/test/test_methodobject.py
b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -63,6 +63,7 @@
),
])
assert mod.getarg_O(1) == 1
+ assert mod.getarg_O.__name__ == "getarg_O"
raises(TypeError, mod.getarg_O)
raises(TypeError, mod.getarg_O, 1, 1)
diff --git a/pypy/objspace/flow/flowcontext.py
b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -185,7 +185,7 @@
class FlowExecutionContext(ExecutionContext):
def __init__(self, space, code, globals, constargs={}, outer_func=None,
- name=None):
+ name=None, is_generator=False):
ExecutionContext.__init__(self, space)
self.code = code
@@ -208,6 +208,7 @@
initialblock = SpamBlock(FrameState(frame).copy())
self.pendingblocks = collections.deque([initialblock])
self.graph = FunctionGraph(name or code.co_name, initialblock)
+ self.is_generator = is_generator
make_link = Link # overridable for transition tracking
@@ -247,6 +248,8 @@
return outcome, w_exc_cls, w_exc_value
def build_flow(self):
+ if self.is_generator:
+ self.produce_generator_mark()
while self.pendingblocks:
block = self.pendingblocks.popleft()
frame = self.create_frame()
@@ -259,9 +262,15 @@
self.topframeref = jit.non_virtual_ref(frame)
self.crnt_frame = frame
try:
- w_result = frame.dispatch(frame.pycode,
- frame.last_instr,
- self)
+ frame.frame_finished_execution = False
+ while True:
+ w_result = frame.dispatch(frame.pycode,
+ frame.last_instr,
+ self)
+ if frame.frame_finished_execution:
+ break
+ else:
+ self.generate_yield(frame, w_result)
finally:
self.crnt_frame = None
self.topframeref = old_frameref
@@ -307,6 +316,21 @@
del self.recorder
self.fixeggblocks()
+ def produce_generator_mark(self):
+ [initialblock] = self.pendingblocks
+ initialblock.operations.append(
+ SpaceOperation('generator_mark', [], Variable()))
+
+ def generate_yield(self, frame, w_result):
+ assert self.is_generator
+ self.recorder.crnt_block.operations.append(
+ SpaceOperation('yield', [w_result], Variable()))
+ # we must push a dummy value that will be POPped: it's the .send()
+ # passed into the generator (2.5 feature)
+ assert sys.version_info >= (2, 5)
+ frame.pushvalue(None)
+ frame.last_instr += 1
+
def fixeggblocks(self):
# EggBlocks reuse the variables of their previous block,
# which is deemed not acceptable for simplicity of the operations
diff --git a/pypy/objspace/flow/objspace.py b/pypy/objspace/flow/objspace.py
--- a/pypy/objspace/flow/objspace.py
+++ b/pypy/objspace/flow/objspace.py
@@ -8,6 +8,7 @@
from pypy.interpreter.pycode import PyCode, cpython_code_signature
from pypy.interpreter.module import Module
from pypy.interpreter.error import OperationError
+from pypy.interpreter.astcompiler.consts import CO_GENERATOR
from pypy.interpreter import pyframe, argument
from pypy.objspace.flow.model import *
from pypy.objspace.flow import flowcontext, operation, specialcase
@@ -247,15 +248,13 @@
return ecls
return None
- def build_flow(self, func, constargs={}):
+ def build_flow(self, func, constargs={}, tweak_for_generator=True):
"""
"""
if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
raise Exception, "%r is tagged as NOT_RPYTHON" % (func,)
code = func.func_code
- if code.co_flags & 32:
- # generator
- raise TypeError("%r is a generator" % (func,))
+ is_generator = bool(code.co_flags & CO_GENERATOR)
code = PyCode._from_code(self, code)
if func.func_closure is None:
cl = None
@@ -271,7 +270,8 @@
class outerfunc: # hack
closure = cl
ec = flowcontext.FlowExecutionContext(self, code, func.func_globals,
- constargs, outerfunc, name)
+ constargs, outerfunc, name,
+ is_generator)
graph = ec.graph
graph.func = func
# attach a signature and defaults to the graph
@@ -291,6 +291,11 @@
e = error.FlowingError(formated)
raise error.FlowingError, e, tb
checkgraph(graph)
+ #
+ if is_generator and tweak_for_generator:
+ from pypy.translator.generator import tweak_generator_graph
+ tweak_generator_graph(graph)
+ #
return graph
def fixedview(self, w_tuple, expected_length=None):
diff --git a/pypy/objspace/flow/test/test_generator.py
b/pypy/objspace/flow/test/test_generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/flow/test/test_generator.py
@@ -0,0 +1,18 @@
+from pypy.objspace.flow.test.test_objspace import Base
+
+
+class TestGenerator(Base):
+
+ def test_simple_generator(self):
+ def f(n):
+ i = 0
+ while i < n:
+ yield i
+ yield i
+ i += 1
+ graph = self.codetest(f, tweak_for_generator=False)
+ ops = self.all_operations(graph)
+ assert ops == {'generator_mark': 1,
+ 'lt': 1, 'is_true': 1,
+ 'yield': 2,
+ 'inplace_add': 1}
diff --git a/pypy/objspace/flow/test/test_objspace.py
b/pypy/objspace/flow/test/test_objspace.py
--- a/pypy/objspace/flow/test/test_objspace.py
+++ b/pypy/objspace/flow/test/test_objspace.py
@@ -16,14 +16,14 @@
is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2
class Base:
- def codetest(self, func):
+ def codetest(self, func, **kwds):
import inspect
try:
func = func.im_func
except AttributeError:
pass
#name = func.func_name
- graph = self.space.build_flow(func)
+ graph = self.space.build_flow(func, **kwds)
graph.source = inspect.getsource(func)
self.show(graph)
return graph
@@ -882,12 +882,6 @@
num = bytecode_spec.opmap[name]
flow_meth_names[num] = locals()['old_' + name]
- def test_generator(self):
- def f():
- yield 3
-
- py.test.raises(TypeError, "self.codetest(f)")
-
def test_dont_capture_RuntimeError(self):
class Foo:
def __hash__(self):
diff --git a/pypy/rpython/test/test_generator.py
b/pypy/rpython/test/test_generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/test/test_generator.py
@@ -0,0 +1,62 @@
+from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
+
+
+class BaseTestGenerator(BaseRtypingTest):
+
+ def test_simple_explicit(self):
+ def g(a, b, c):
+ yield a
+ yield b
+ yield c
+ def f():
+ gen = g(3, 5, 8)
+ x = gen.next() * 100
+ x += gen.next() * 10
+ x += gen.next()
+ return x
+ res = self.interpret(f, [])
+ assert res == 358
+
+ def test_cannot_merge(self):
+ # merging two different generators is not supported
+ # right now, but we can use workarounds like here
+ class MyGen:
+ _immutable_ = True
+ def next(self):
+ raise NotImplementedError
+ class MyG1(MyGen):
+ _immutable_ = True
+ def __init__(self, a):
+ self._gen = self.g1(a)
+ def next(self):
+ return self._gen.next()
+ @staticmethod
+ def g1(a):
+ yield a + 1
+ yield a + 2
+ class MyG2(MyGen):
+ _immutable_ = True
+ def __init__(self):
+ self._gen = self.g2()
+ def next(self):
+ return self._gen.next()
+ @staticmethod
+ def g2():
+ yield 42
+ def f(n):
+ if n > 0:
+ gen = MyG1(n)
+ else:
+ gen = MyG2()
+ return gen.next()
+ res = self.interpret(f, [10])
+ assert res == 11
+ res = self.interpret(f, [0])
+ assert res == 42
+
+
+class TestLLtype(BaseTestGenerator, LLRtypeMixin):
+ pass
+
+class TestOOtype(BaseTestGenerator, OORtypeMixin):
+ pass
diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py
--- a/pypy/tool/jitlogparser/parser.py
+++ b/pypy/tool/jitlogparser/parser.py
@@ -3,6 +3,7 @@
from pypy.jit.metainterp.resoperation import opname
from pypy.jit.tool.oparser import OpParser
from pypy.tool.logparser import parse_log_file, extract_category
+from copy import copy
class Op(object):
bridge = None
@@ -387,6 +388,18 @@
loops.append(loop)
return log, loops
+def split_trace(trace):
+ labels = [i for i, op in enumerate(trace.operations)
+ if op.name == 'label']
+ labels = [0] + labels + [len(trace.operations) - 1]
+ parts = []
+ for i in range(len(labels) - 1):
+ start, stop = labels[i], labels[i+1]
+ part = copy(trace)
+ part.operations = trace.operations[start : stop + 1]
+ parts.append(part)
+
+ return parts
def parse_log_counts(input, loops):
if not input:
diff --git a/pypy/tool/jitlogparser/test/test_modulefinder.py
b/pypy/tool/jitlogparser/test/test_modulefinder.py
--- a/pypy/tool/jitlogparser/test/test_modulefinder.py
+++ b/pypy/tool/jitlogparser/test/test_modulefinder.py
@@ -7,12 +7,14 @@
py.test.skip("Specific python 2.6 tests")
def test_gather_code_py():
+ py.test.skip("XXX broken, fix me")
fname = re.__file__
codes = gather_all_code_objs(fname)
assert len(codes) == 21
assert sorted(codes.keys()) == [102, 134, 139, 144, 153, 164, 169, 181,
188, 192, 197, 206, 229, 251, 266, 271, 277, 285, 293, 294, 308]
def test_load_code():
+ py.test.skip("XXX broken, fix me")
fname = re.__file__
code = gather_all_code_objs(fname)[144]
assert code.co_name == 'sub'
diff --git a/pypy/tool/jitlogparser/test/test_parser.py
b/pypy/tool/jitlogparser/test/test_parser.py
--- a/pypy/tool/jitlogparser/test/test_parser.py
+++ b/pypy/tool/jitlogparser/test/test_parser.py
@@ -1,6 +1,6 @@
from pypy.tool.jitlogparser.parser import (SimpleParser, TraceForOpcode,
Function, adjust_bridges,
- import_log, Op)
+ import_log, split_trace, Op)
from pypy.tool.jitlogparser.storage import LoopStorage
import py, sys
@@ -231,3 +231,21 @@
myrepr = 'c = foobar(a, b, descr=mydescr)'
assert op.repr() == myrepr
assert op.repr() == myrepr # do it twice
+
+def test_split_trace():
+ loop = parse('''
+ [i7]
+ i9 = int_lt(i7, 1003)
+ label(i9)
+ guard_true(i9, descr=<Guard2>) []
+ i13 = getfield_raw(151937600, descr=<SignedFieldDescr
pypysig_long_struct.c_value 0>)
+ label(i13)
+ i19 = int_lt(i13, 1003)
+ guard_true(i19, descr=<Guard2>) []
+ i113 = getfield_raw(151937600, descr=<SignedFieldDescr
pypysig_long_struct.c_value 0>)
+ ''')
+ parts = split_trace(loop)
+ assert len(parts) == 3
+ assert len(parts[0].operations) == 2
+ assert len(parts[1].operations) == 4
+ assert len(parts[2].operations) == 4
diff --git a/pypy/translator/generator.py b/pypy/translator/generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/generator.py
@@ -0,0 +1,166 @@
+from pypy.objspace.flow.model import Block, Link, SpaceOperation, checkgraph
+from pypy.objspace.flow.model import Variable, Constant, FunctionGraph
+from pypy.translator.unsimplify import insert_empty_startblock
+from pypy.translator.unsimplify import split_block
+from pypy.translator.simplify import eliminate_empty_blocks
+from pypy.tool.sourcetools import func_with_new_name
+from pypy.interpreter.argument import Signature
+
+
+class AbstractPosition(object):
+ _immutable_ = True
+ _attrs_ = ()
+
+
+def tweak_generator_graph(graph):
+ if not hasattr(graph.func, '_generator_next_method_of_'):
+ # This is the first copy of the graph. We replace it with
+ # a small bootstrap graph.
+ GeneratorIterator = make_generatoriterator_class(graph)
+ replace_graph_with_bootstrap(GeneratorIterator, graph)
+ # We attach a 'next' method to the GeneratorIterator class
+ # that will invoke the real function, based on a second
+ # copy of the graph.
+ attach_next_method(GeneratorIterator, graph)
+ else:
+ # This is the second copy of the graph. Tweak it.
+ GeneratorIterator = graph.func._generator_next_method_of_
+ tweak_generator_body_graph(GeneratorIterator.Entry, graph)
+
+
+def make_generatoriterator_class(graph):
+ class GeneratorIterator(object):
+ class Entry(AbstractPosition):
+ _immutable_ = True
+ varnames = get_variable_names(graph.startblock.inputargs)
+ def __init__(self, entry):
+ self.current = entry
+ return GeneratorIterator
+
+def replace_graph_with_bootstrap(GeneratorIterator, graph):
+ Entry = GeneratorIterator.Entry
+ newblock = Block(graph.startblock.inputargs)
+ v_generator = Variable('generator')
+ v_entry = Variable('entry')
+ newblock.operations.append(
+ SpaceOperation('simple_call', [Constant(Entry)], v_entry))
+ assert len(graph.startblock.inputargs) == len(Entry.varnames)
+ for v, name in zip(graph.startblock.inputargs, Entry.varnames):
+ newblock.operations.append(
+ SpaceOperation('setattr', [v_entry, Constant(name), v],
+ Variable()))
+ newblock.operations.append(
+ SpaceOperation('simple_call', [Constant(GeneratorIterator), v_entry],
+ v_generator))
+ newblock.closeblock(Link([v_generator], graph.returnblock))
+ graph.startblock = newblock
+
+def attach_next_method(GeneratorIterator, graph):
+ func = graph.func
+ func = func_with_new_name(func, '%s__next' % (func.func_name,))
+ func._generator_next_method_of_ = GeneratorIterator
+ func._always_inline_ = True
+ #
+ def next(self):
+ entry = self.current
+ self.current = None
+ (next_entry, return_value) = func(entry)
+ self.current = next_entry
+ return return_value
+ GeneratorIterator.next = next
+ return func # for debugging
+
+def get_variable_names(variables):
+ seen = set()
+ result = []
+ for v in variables:
+ name = v._name.strip('_')
+ while name in seen:
+ name += '_'
+ result.append('g_' + name)
+ seen.add(name)
+ return result
+
+def _insert_reads(block, varnames):
+ assert len(varnames) == len(block.inputargs)
+ v_entry1 = Variable('entry')
+ for i, name in enumerate(varnames):
+ block.operations.insert(i,
+ SpaceOperation('getattr', [v_entry1, Constant(name)],
+ block.inputargs[i]))
+ block.inputargs = [v_entry1]
+
+def tweak_generator_body_graph(Entry, graph):
+ assert graph.startblock.operations[0].opname == 'generator_mark'
+ graph.startblock.operations.pop(0)
+ #
+ insert_empty_startblock(None, graph)
+ _insert_reads(graph.startblock, Entry.varnames)
+ Entry.block = graph.startblock
+ #
+ mappings = [Entry]
+ #
+ for block in list(graph.iterblocks()):
+ for exit in block.exits:
+ if exit.target is graph.returnblock:
+ exit.args = [Constant(StopIteration),
+ Constant(StopIteration())]
+ exit.target = graph.exceptblock
+ for index in range(len(block.operations)-1, -1, -1):
+ op = block.operations[index]
+ if op.opname == 'yield':
+ [v_yielded_value] = op.args
+ del block.operations[index]
+ newlink = split_block(None, block, index)
+ newblock = newlink.target
+ #
+ class Resume(AbstractPosition):
+ _immutable_ = True
+ block = newblock
+ Resume.__name__ = 'Resume%d' % len(mappings)
+ mappings.append(Resume)
+ varnames = get_variable_names(newlink.args)
+ #
+ _insert_reads(newblock, varnames)
+ #
+ v_resume = Variable('resume')
+ block.operations.append(
+ SpaceOperation('simple_call', [Constant(Resume)],
+ v_resume))
+ for i, name in enumerate(varnames):
+ block.operations.append(
+ SpaceOperation('setattr', [v_resume, Constant(name),
+ newlink.args[i]],
+ Variable()))
+ v_pair = Variable('pair')
+ block.operations.append(
+ SpaceOperation('newtuple', [v_resume, v_yielded_value],
+ v_pair))
+ newlink.args = [v_pair]
+ newlink.target = graph.returnblock
+ #
+ regular_entry_block = Block([Variable('entry')])
+ block = regular_entry_block
+ for Resume in mappings:
+ v_check = Variable()
+ block.operations.append(
+ SpaceOperation('simple_call', [Constant(isinstance),
+ block.inputargs[0],
+ Constant(Resume)],
+ v_check))
+ block.exitswitch = v_check
+ link1 = Link([block.inputargs[0]], Resume.block)
+ link1.exitcase = True
+ nextblock = Block([Variable('entry')])
+ link2 = Link([block.inputargs[0]], nextblock)
+ link2.exitcase = False
+ block.closeblock(link1, link2)
+ block = nextblock
+ block.closeblock(Link([Constant(AssertionError),
+ Constant(AssertionError("bad generator class"))],
+ graph.exceptblock))
+ graph.startblock = regular_entry_block
+ graph.signature = Signature(['entry'])
+ graph.defaults = ()
+ checkgraph(graph)
+ eliminate_empty_blocks(graph)
diff --git a/pypy/translator/test/test_generator.py
b/pypy/translator/test/test_generator.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/test/test_generator.py
@@ -0,0 +1,156 @@
+from pypy.conftest import option
+from pypy.objspace.flow.objspace import FlowObjSpace
+from pypy.objspace.flow.model import Variable
+from pypy.interpreter.argument import Signature
+from pypy.translator.translator import TranslationContext
+from pypy.translator.generator import make_generatoriterator_class
+from pypy.translator.generator import replace_graph_with_bootstrap
+from pypy.translator.generator import get_variable_names
+from pypy.translator.generator import tweak_generator_body_graph
+from pypy.translator.generator import attach_next_method
+from pypy.translator.simplify import join_blocks
+
+
+# ____________________________________________________________
+
+def f_gen(n):
+ i = 0
+ while i < n:
+ yield i
+ i += 1
+
+class GeneratorIterator(object):
+ def __init__(self, entry):
+ self.current = entry
+ def next(self):
+ e = self.current
+ self.current = None
+ if isinstance(e, Yield1):
+ n = e.n_0
+ i = e.i_0
+ i += 1
+ else:
+ n = e.n_0
+ i = 0
+ if i < n:
+ e = Yield1()
+ e.n_0 = n
+ e.i_0 = i
+ self.current = e
+ return i
+ raise StopIteration
+
+ def __iter__(self):
+ return self
+
+class AbstractPosition(object):
+ _immutable_ = True
+class Entry1(AbstractPosition):
+ _immutable_ = True
+class Yield1(AbstractPosition):
+ _immutable_ = True
+
+def f_explicit(n):
+ e = Entry1()
+ e.n_0 = n
+ return GeneratorIterator(e)
+
+def test_explicit():
+ assert list(f_gen(10)) == list(f_explicit(10))
+
+def test_get_variable_names():
+ lst = get_variable_names([Variable('a'), Variable('b_'), Variable('a')])
+ assert lst == ['g_a', 'g_b', 'g_a_']
+
+# ____________________________________________________________
+
+
+class TestGenerator:
+
+ def test_replace_graph_with_bootstrap(self):
+ def func(n, x, y, z):
+ yield n
+ yield n
+ #
+ space = FlowObjSpace()
+ graph = space.build_flow(func, tweak_for_generator=False)
+ assert graph.startblock.operations[0].opname == 'generator_mark'
+ GeneratorIterator = make_generatoriterator_class(graph)
+ replace_graph_with_bootstrap(GeneratorIterator, graph)
+ if option.view:
+ graph.show()
+ block = graph.startblock
+ ops = block.operations
+ assert ops[0].opname == 'simple_call' # e = Entry1()
+ assert ops[1].opname == 'setattr' # e.g_n = n
+ assert ops[1].args[1].value == 'g_n'
+ assert ops[2].opname == 'setattr' # e.g_x = x
+ assert ops[2].args[1].value == 'g_x'
+ assert ops[3].opname == 'setattr' # e.g_y = y
+ assert ops[3].args[1].value == 'g_y'
+ assert ops[4].opname == 'setattr' # e.g_z = z
+ assert ops[4].args[1].value == 'g_z'
+ assert ops[5].opname == 'simple_call' # g = GeneratorIterator(e)
+ assert ops[5].args[1] == ops[0].result
+ assert len(ops) == 6
+ assert len(block.exits) == 1
+ assert block.exits[0].target is graph.returnblock
+
+ def test_tweak_generator_body_graph(self):
+ def f(n, x, y, z=3):
+ z *= 10
+ yield n + 1
+ z -= 10
+ #
+ space = FlowObjSpace()
+ graph = space.build_flow(f, tweak_for_generator=False)
+ class Entry:
+ varnames = ['g_n', 'g_x', 'g_y', 'g_z']
+ tweak_generator_body_graph(Entry, graph)
+ if option.view:
+ graph.show()
+ # XXX how to test directly that the graph is correct? :-(
+ assert len(graph.startblock.inputargs) == 1
+ assert graph.signature == Signature(['entry'])
+ assert graph.defaults == ()
+
+ def test_tweak_generator_graph(self):
+ def f(n, x, y, z):
+ z *= 10
+ yield n + 1
+ z -= 10
+ #
+ space = FlowObjSpace()
+ graph = space.build_flow(f, tweak_for_generator=False)
+ GeneratorIterator = make_generatoriterator_class(graph)
+ replace_graph_with_bootstrap(GeneratorIterator, graph)
+ func1 = attach_next_method(GeneratorIterator, graph)
+ if option.view:
+ graph.show()
+ #
+ assert func1._generator_next_method_of_ is GeneratorIterator
+ assert hasattr(GeneratorIterator, 'next')
+ #
+ graph_next = space.build_flow(GeneratorIterator.next.im_func)
+ join_blocks(graph_next)
+ if option.view:
+ graph_next.show()
+ #
+ graph1 = space.build_flow(func1, tweak_for_generator=False)
+ tweak_generator_body_graph(GeneratorIterator.Entry, graph1)
+ if option.view:
+ graph1.show()
+
+ def test_automatic(self):
+ def f(n, x, y, z):
+ z *= 10
+ yield n + 1
+ z -= 10
+ #
+ space = FlowObjSpace()
+ graph = space.build_flow(f) # tweak_for_generator=True
+ if option.view:
+ graph.show()
+ block = graph.startblock
+ assert len(block.exits) == 1
+ assert block.exits[0].target is graph.returnblock
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit
