Author: Matti Picus <matti.pi...@gmail.com> Branch: py3.5 Changeset: r93886:6dcfe4a00bb7 Date: 2018-02-25 20:24 -0500 http://bitbucket.org/pypy/pypy/changeset/6dcfe4a00bb7/
Log: merge default into branch diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py --- a/lib_pypy/_dbm.py +++ b/lib_pypy/_dbm.py @@ -168,7 +168,14 @@ def open(filename, flag='r', mode=0o666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") filename = filename.encode(sys.getdefaultencoding()) openflag = 0 diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -567,6 +567,11 @@ versions of PyPy may have to rename the arguments if CPython starts accepting them too. +* PyPy3: `distutils` has been enhanced to allow finding `VsDevCmd.bat` in the + directory pointed to by the `VS%0.f0COMNTOOLS` (typically `VS140COMNTOOLS`) + environment variable. CPython searches for `vcvarsall.bat` somewhere _above_ + that value. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,7 @@ .. branch: winapi Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 + +.. branch: refactor-slots + +Refactor cpyext slots. diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -385,8 +385,11 @@ if not (last_pos == ctx.match_start == ctx.match_end and n > 0): # the above ignores empty matches on latest position + last_pos = ctx.match_end if filter_is_callable: w_match = self.getmatch(ctx, True) + # make a copy of 'ctx'; see test_sub_matches_stay_valid + ctx = ctx.fresh_copy(start) # match_start/match_end dropped w_piece = space.call_function(w_filter, w_match) if not space.is_w(w_piece, space.w_None): assert strbuilder is None and unicodebuilder is None @@ -403,7 +406,6 @@ unicodebuilder.append(filter_as_unicode) else: sublist_w.append(w_filter) - last_pos = ctx.match_end n += 1 elif last_pos >= ctx.end: break # empty match at the end: finished diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -355,6 +355,18 @@ KEYCRE = re.compile(r"%\(([^)]*)\)s|.") raises(TypeError, KEYCRE.sub, "hello", {"%(": 1}) + def test_sub_matches_stay_valid(self): + import re + matches = [] + def callback(match): + matches.append(match) + return "x" + result = re.compile(r"[ab]").sub(callback, "acb") + assert result == "xcx" + assert len(matches) == 2 + assert matches[0].group() == "a" + assert matches[1].group() == "b" + class AppTestSreScanner: diff --git a/pypy/module/test_lib_pypy/test_dbm_extra.py b/pypy/module/test_lib_pypy/test_dbm_extra.py --- a/pypy/module/test_lib_pypy/test_dbm_extra.py +++ b/pypy/module/test_lib_pypy/test_dbm_extra.py @@ -1,4 +1,4 @@ -import py +import py, os from rpython.tool.udir import udir try: from lib_pypy import dbm @@ -73,3 +73,8 @@ assert 'key_with_empty_value' in d assert d['key_with_empty_value'] == '' d.close() + +def test_unicode_filename(): + path = str(udir) + os.sep + u'test_dbm_extra.test_unicode_filename' + d = dbm.open(path, 'c') + d.close() diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -354,7 +354,7 @@ else: # this is dead code, but in case we have a gc that does # not have a write barrier and does not zero memory, we would - # need to clal it + # need to call it if op.getopnum() == rop.SETFIELD_GC: self.consider_setfield_gc(op) elif op.getopnum() == rop.SETARRAYITEM_GC: diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -30,7 +30,7 @@ class CompileData(object): memo = None log_noopt = True - + def forget_optimization_info(self): for arg in self.trace.inputargs: arg.set_forwarded(None) @@ -67,19 +67,26 @@ """ This represents label() ops jump with no extra info associated with the label """ - def __init__(self, trace, call_pure_results=None, + def __init__(self, trace, resumestorage=None, call_pure_results=None, enable_opts=None): self.trace = trace + self.resumestorage = resumestorage self.call_pure_results = call_pure_results self.enable_opts = enable_opts def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer + from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge #assert not unroll opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.propagate_all_forward(self.trace.get_iter(), - self.call_pure_results) + traceiter = self.trace.get_iter() + if self.resumestorage: + frontend_inputargs = self.trace.inputargs + deserialize_optimizer_knowledge(opt, self.resumestorage, + frontend_inputargs, + traceiter.inputargs) + return opt.propagate_all_forward(traceiter, self.call_pure_results) class BridgeCompileData(CompileData): """ This represents ops() with a jump at the end that goes to some @@ -518,7 +525,7 @@ for item in lst: item.set_forwarded(None) # XXX we should really do it, but we need to remember the values - # somehoe for ContinueRunningNormally + # somehow for ContinueRunningNormally if reset_values: item.reset_value() @@ -671,38 +678,16 @@ raise jitexc.ExitFrameWithExceptionRef(cpu, value) -class TerminatingLoopToken(JitCellToken): # FIXME: kill? - terminating = True - - def __init__(self, nargs, finishdescr): - self.finishdescr = finishdescr - -def make_done_loop_tokens(): - done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid() - done_with_this_frame_descr_int = DoneWithThisFrameDescrInt() - done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef() - done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat() - exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() - - # pseudo loop tokens to make the life of optimize.py easier - d = {'loop_tokens_done_with_this_frame_int': [ - TerminatingLoopToken(1, done_with_this_frame_descr_int) - ], - 'loop_tokens_done_with_this_frame_ref': [ - TerminatingLoopToken(1, done_with_this_frame_descr_ref) - ], - 'loop_tokens_done_with_this_frame_float': [ - TerminatingLoopToken(1, done_with_this_frame_descr_float) - ], - 'loop_tokens_done_with_this_frame_void': [ - TerminatingLoopToken(0, done_with_this_frame_descr_void) - ], - 'loop_tokens_exit_frame_with_exception_ref': [ - TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) - ], - } - d.update(locals()) - return d +def make_and_attach_done_descrs(targets): + for name, cls in [ + ("done_with_this_frame_descr_void", DoneWithThisFrameDescrVoid), + ("done_with_this_frame_descr_int", DoneWithThisFrameDescrInt), + ("done_with_this_frame_descr_ref", DoneWithThisFrameDescrRef), + ("done_with_this_frame_descr_float", DoneWithThisFrameDescrFloat), + ("exit_frame_with_exception_descr_ref", ExitFrameWithExceptionDescrRef)]: + descr = cls() + for target in targets: + setattr(target, name, descr) class ResumeDescr(AbstractFailDescr): _attrs_ = () @@ -726,6 +711,9 @@ TY_REF = 0x04 TY_FLOAT = 0x06 + def get_resumestorage(self): + raise NotImplementedError("abstract base class") + def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): if (self.must_compile(deadframe, metainterp_sd, jitdriver_sd) and not rstack.stack_almost_full()): @@ -854,15 +842,23 @@ class ResumeGuardCopiedDescr(AbstractResumeGuardDescr): _attrs_ = ('status', 'prev') + def __init__(self, prev): + AbstractResumeGuardDescr.__init__(self) + assert isinstance(prev, ResumeGuardDescr) + self.prev = prev + def copy_all_attributes_from(self, other): assert isinstance(other, ResumeGuardCopiedDescr) self.prev = other.prev def clone(self): - cloned = ResumeGuardCopiedDescr() - cloned.copy_all_attributes_from(self) + cloned = ResumeGuardCopiedDescr(self.prev) return cloned + def get_resumestorage(self): + prev = self.prev + assert isinstance(prev, ResumeGuardDescr) + return prev class ResumeGuardDescr(AbstractResumeGuardDescr): _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals', @@ -873,8 +869,7 @@ rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) def copy_all_attributes_from(self, other): - if isinstance(other, ResumeGuardCopiedDescr): - other = other.prev + other = other.get_resumestorage() assert isinstance(other, ResumeGuardDescr) self.rd_consts = other.rd_consts self.rd_pendingfields = other.rd_pendingfields @@ -895,6 +890,9 @@ cloned.copy_all_attributes_from(self) return cloned + def get_resumestorage(self): + return self + class ResumeGuardExcDescr(ResumeGuardDescr): pass @@ -936,22 +934,22 @@ ptr = cpu.ts.cast_to_baseclass(gcref) return cast_base_ptr_to_instance(AllVirtuals, ptr) -def invent_fail_descr_for_op(opnum, optimizer, copied_guard=False): +def invent_fail_descr_for_op(opnum, optimizer, copied_from_descr=None): if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: - assert not copied_guard + assert copied_from_descr is None resumedescr = ResumeGuardForcedDescr() resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd) elif opnum in (rop.GUARD_IS_OBJECT, rop.GUARD_SUBCLASS, rop.GUARD_GC_TYPE): # note - this only happens in tests resumedescr = ResumeAtPositionDescr() elif opnum in (rop.GUARD_EXCEPTION, rop.GUARD_NO_EXCEPTION): - if copied_guard: - resumedescr = ResumeGuardCopiedExcDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedExcDescr(copied_from_descr) else: resumedescr = ResumeGuardExcDescr() else: - if copied_guard: - resumedescr = ResumeGuardCopiedDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedDescr(copied_from_descr) else: resumedescr = ResumeGuardDescr() return resumedescr @@ -1036,6 +1034,9 @@ self.original_greenkey, jitcell_token) metainterp_sd.stats.add_jitcell_token(jitcell_token) + def get_resumestorage(self): + return None + def compile_trace(metainterp, resumekey, runtime_boxes): """Try to compile a new bridge leading from the beginning of the history @@ -1067,22 +1068,15 @@ enable_opts = jitdriver_sd.warmstate.enable_opts call_pure_results = metainterp.call_pure_results + resumestorage = resumekey.get_resumestorage() if metainterp.history.ends_with_jump: - if isinstance(resumekey, ResumeGuardCopiedDescr): - key = resumekey.prev - assert isinstance(key, ResumeGuardDescr) - elif isinstance(resumekey, ResumeFromInterpDescr): - key = None - else: - key = resumekey - assert isinstance(key, ResumeGuardDescr) - data = BridgeCompileData(trace, runtime_boxes, key, + data = BridgeCompileData(trace, runtime_boxes, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts, inline_short_preamble=inline_short_preamble) else: - data = SimpleCompileData(trace, + data = SimpleCompileData(trace, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts) try: diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -404,7 +404,6 @@ target_tokens = None failed_states = None retraced_count = 0 - terminating = False # see TerminatingLoopToken in compile.py invalidated = False outermost_jitdriver_sd = None # and more data specified by the backend when the loop is compiled @@ -935,7 +934,7 @@ return insns def check_simple_loop(self, expected=None, **check): - """ Usefull in the simplest case when we have only one trace ending with + """ Useful in the simplest case when we have only one trace ending with a jump back to itself and possibly a few bridges. Only the operations within the loop formed by that single jump will be counted. diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -688,12 +688,10 @@ def _copy_resume_data_from(self, guard_op, last_guard_op): - descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, True) last_descr = last_guard_op.getdescr() + descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, last_descr) assert isinstance(last_descr, compile.ResumeGuardDescr) - if isinstance(descr, compile.ResumeGuardCopiedDescr): - descr.prev = last_descr - else: + if not isinstance(descr, compile.ResumeGuardCopiedDescr): descr.copy_all_attributes_from(last_descr) guard_op.setdescr(descr) guard_op.setfailargs(last_guard_op.getfailargs()) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -31,8 +31,8 @@ expected = convert_old_style_to_targets(exp, jump=True) call_pure_results = self._convert_call_pure_results(call_pure_results) trace = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu)) - compile_data = compile.SimpleCompileData(trace, - call_pure_results) + compile_data = compile.SimpleCompileData( + trace, call_pure_results=call_pure_results) info, ops = self._do_optimize_loop(compile_data) label_op = ResOperation(rop.LABEL, info.inputargs) loop.inputargs = info.inputargs diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1854,12 +1854,7 @@ self._addr2name_keys = [] self._addr2name_values = [] - self.__dict__.update(compile.make_done_loop_tokens()) - for val in ['int', 'float', 'ref', 'void']: - fullname = 'done_with_this_frame_descr_' + val - setattr(self.cpu, fullname, getattr(self, fullname)) - d = self.exit_frame_with_exception_descr_ref - self.cpu.exit_frame_with_exception_descr_ref = d + compile.make_and_attach_done_descrs([self, cpu]) def _freeze_(self): return True @@ -1909,8 +1904,8 @@ history.REF: 'ref', history.FLOAT: 'float', history.VOID: 'void'}[jd.result_type] - tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) - jd.portal_finishtoken = tokens[0].finishdescr + token = getattr(self, 'done_with_this_frame_descr_%s' % name) + jd.portal_finishtoken = token jd.propagate_exc_descr = exc_descr # self.cpu.propagate_exception_descr = exc_descr @@ -2463,10 +2458,7 @@ def handle_guard_failure(self, resumedescr, deadframe): debug_start('jit-tracing') self.staticdata.profiler.start_tracing() - if isinstance(resumedescr, compile.ResumeGuardCopiedDescr): - key = resumedescr.prev - else: - key = resumedescr + key = resumedescr.get_resumestorage() assert isinstance(key, compile.ResumeGuardDescr) # store the resumekey.wref_original_loop_token() on 'self' to make # sure that it stays alive as long as this MetaInterp @@ -2770,21 +2762,19 @@ if result_type == history.VOID: assert exitbox is None exits = [] - loop_tokens = sd.loop_tokens_done_with_this_frame_void + token = sd.done_with_this_frame_descr_void elif result_type == history.INT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_int + token = sd.done_with_this_frame_descr_int elif result_type == history.REF: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_ref + token = sd.done_with_this_frame_descr_ref elif result_type == history.FLOAT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_float + token = sd.done_with_this_frame_descr_float else: assert False - # FIXME: kill TerminatingLoopToken? # FIXME: can we call compile_trace? - token = loop_tokens[0].finishdescr self.history.record(rop.FINISH, exits, None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, exits) @@ -2810,7 +2800,7 @@ def compile_exit_frame_with_exception(self, valuebox): self.store_token_in_vable() sd = self.staticdata - token = sd.loop_tokens_exit_frame_with_exception_ref[0].finishdescr + token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -143,6 +143,42 @@ self.check_trace_count(3) self.check_resops(guard_class=1) + def test_bridge_guard_class_return(self): + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) + class A(object): + def f(self): + return 1 + class B(A): + def f(self): + return 2 + def f(x, y, n): + if x: + a = A() + else: + a = B() + a.x = 0 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a) + res += a.f() + a.x += 1 + if y < n: + res += 1 + res += a.f() + return res + res += a.f() + y -= 1 + return res + def g(i): + res = 0 + for i in range(i): + res += f(6, 32, 16-i) + res1 = g(10) + res2 = self.meta_interp(g, [10]) + assert res1 == res2 + self.check_trace_count(2) + self.check_resops(guard_class=1, omit_finish=False) + def test_bridge_field_read(self): myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) class A(object): diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -137,14 +137,11 @@ def decorator(f): def get_annotation(t): from rpython.annotator.signature import annotation - from rpython.annotator.model import SomeObject, SomeString, SomeUnicodeString + from rpython.annotator.model import SomeObject if isinstance(t, SomeObject): return t - s_result = annotation(t) - if (isinstance(s_result, SomeString) or - isinstance(s_result, SomeUnicodeString)): - return s_result.__class__(can_be_None=True) - return s_result + return annotation(t) + def get_type_descr_of_argument(arg): # we don't want to check *all* the items in list/dict: we assume # they are already homogeneous, so we only check the first diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -109,9 +109,9 @@ wchar_t const* file, unsigned int line, uintptr_t pReserved) { - wprintf(L"Invalid parameter detected in function %s." - L" File: %s Line: %d\\n", function, file, line); - wprintf(L"Expression: %s\\n", expression); + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); } RPY_EXTERN void* enter_suppress_iph(void) @@ -267,7 +267,7 @@ if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], rffi.INT, compilation_info=errno_eci, )) c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", @@ -515,7 +515,7 @@ releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('read') -@enforceargs(int, int) +@signature(types.int(), types.int(), returns=types.any()) def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) @@ -526,7 +526,7 @@ return buf.str(got) @replace_os_function('write') -@enforceargs(int, None) +@signature(types.int(), types.any(), returns=types.any()) def write(fd, data): count = len(data) with FdValidator(fd): @@ -536,6 +536,7 @@ return handle_posix_error('write', ret) @replace_os_function('close') +@signature(types.int(), returns=types.any()) def close(fd): with FdValidator(fd): handle_posix_error('close', c_close(fd)) diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -484,12 +484,6 @@ # in RPython there is an implicit int->float promotion assert f(42) == 42 -def test_enforceargs_None_string(): - @enforceargs(str, unicode) - def f(a, b): - return a, b - assert f(None, None) == (None, None) - def test_enforceargs_complex_types(): @enforceargs([int], {str: int}) def f(a, b): _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit