Author: Maciej Fijalkowski <fij...@gmail.com> Branch: Changeset: r79359:ef6c662bf331 Date: 2015-09-01 17:31 +0200 http://bitbucket.org/pypy/pypy/changeset/ef6c662bf331/
Log: merge diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -15,3 +15,4 @@ e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0 e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0 295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0 +f3ad1e1e1d6215e20d34bb65ab85ff9188c9f559 release-2.6.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -168,7 +168,6 @@ Michael Twomey Lucian Branescu Mihaila Yichao Yu - Anton Gulenko Gabriel Lavoie Olivier Dormond Jared Grubb @@ -215,6 +214,7 @@ Carl Meyer Karl Ramm Pieter Zieschang + Anton Gulenko Gabriel Lukas Vacek Andrew Dalke @@ -247,6 +247,7 @@ Toni Mattis Lucas Stadler Julian Berman + Markus Holtermann roberto@goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -429,7 +430,7 @@ gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed under the terms of the GPL license as well. -License for 'pypy/module/_vmprof/src' +License for 'rpython/rlib/rvmprof/src' -------------------------------------- The code is based on gperftools. You may see a copy of the License for it at diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.7 +Version: 0.4.9 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.7" +__version__ = "0.4.9" # ____________________________________________________________ # Exceptions diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -39,7 +39,8 @@ "_csv", "cppyy", "_pypyjson" ]) -if sys.platform.startswith('linux') and os.uname()[4] == 'x86_64': +if (sys.platform.startswith('linux') and os.uname()[4] == 'x86_64' + and sys.maxint > 2**32): # it's not enough that we get x86_64 working_modules.add('_vmprof') translation_modules = default_modules.copy() diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -67,7 +67,7 @@ # The short X.Y version. version = '2.6' # The full version, including alpha/beta/rc tags. -release = '2.6.0' +release = '2.6.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -32,6 +32,7 @@ Lukas Diekmann Sven Hager Anders Lehmann + Richard Plangger Aurelien Campeas Remi Meier Niklaus Haldimann @@ -57,7 +58,6 @@ Ludovic Aubry Jacob Hallen Jason Creighton - Richard Plangger Alex Martelli Michal Bendowski stian @@ -138,7 +138,6 @@ Michael Twomey Lucian Branescu Mihaila Yichao Yu - Anton Gulenko Gabriel Lavoie Olivier Dormond Jared Grubb @@ -185,6 +184,7 @@ Carl Meyer Karl Ramm Pieter Zieschang + Anton Gulenko Gabriel Lukas Vacek Andrew Dalke @@ -217,6 +217,7 @@ Toni Mattis Lucas Stadler Julian Berman + Markus Holtermann roberto@goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -252,6 +253,7 @@ shoma hosaka Daniel Neuhäuser Ben Mather + Niclas Olofsson halgari Boglarka Vezer Chris Pressey diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.6.1.rst release-2.6.0.rst release-2.5.1.rst release-2.5.0.rst diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.6.1.rst @@ -0,0 +1,129 @@ +========== +PyPy 2.6.1 +========== + +We're pleased to announce PyPy 2.6.1, an update to PyPy 2.6.0 released June 1. +We have updated stdlib to 2.7.10, `cffi`_ to version 1.3, extended support for +the new vmprof_ statistical profiler for multiple threads, and increased +functionality of numpy. + +You can download the PyPy 2.6.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and our volunteers and contributors. + +.. _`cffi`: https://cffi.readthedocs.org + +We would also like to encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making +RPython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows 32, OpenBSD_, freebsd_), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _OpenBSD: http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/lang/pypy +.. _freebsd: https://svnweb.freebsd.org/ports/head/lang/pypy/ +.. _`dynamic languages`: http://pypyjs.org + +Highlights +=========== + +* Bug Fixes + + * Revive non-SSE2 support + + * Fixes for detaching _io.Buffer* + + * On Windows, close (and flush) all open sockets on exiting + + * Drop support for ancient macOS v10.4 and before + + * Clear up contention in the garbage collector between trace-me-later and pinning + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +* New features: + + * cffi was updated to version 1.3 + + * The python stdlib was updated to 2.7.10 from 2.7.9 + + * vmprof now supports multiple threads and OS X + + * The translation process builds cffi import libraries for some stdlib + packages, which should prevent confusion when package.py is not used + + * better support for gdb debugging + + * freebsd should be able to translate PyPy "out of the box" with no patches + +* Numpy: + + * Better support for record dtypes, including the ``align`` keyword + + * Implement casting and create output arrays accordingly (still missing some corner cases) + + * Support creation of unicode ndarrays + + * Better support ndarray.flags + + * Support ``axis`` argument in more functions + + * Refactor array indexing to support ellipses + + * Allow the docstrings of built-in numpy objects to be set at run-time + + * Support the ``buffered`` nditer creation keyword + +* Performance improvements: + + * Delay recursive calls to make them non-recursive + + * Skip loop unrolling if it compiles too much code + + * Tweak the heapcache + + * Add a list strategy for lists that store both floats and 32-bit integers. + The latter are encoded as nonstandard NaNs. Benchmarks show that the speed + of such lists is now very close to the speed of purely-int or purely-float + lists. + + * Simplify implementation of ffi.gc() to avoid most weakrefs + + * Massively improve the performance of map() with more than + one sequence argument + +.. _`vmprof`: https://vmprof.readthedocs.org +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.1.html + +Please try it out and let us know what you think. We welcome +success stories, `experiments`_, or `benchmarks`_, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + +.. _`experiments`: https://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html +.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0 diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py --- a/pypy/interpreter/miscutils.py +++ b/pypy/interpreter/miscutils.py @@ -9,6 +9,7 @@ implementation for this feature, and patches 'space.threadlocals' when 'thread' is initialized. """ + _immutable_fields_ = ['_value?'] _value = None def get_ec(self): diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py --- a/pypy/objspace/std/celldict.py +++ b/pypy/objspace/std/celldict.py @@ -3,7 +3,7 @@ indirection is introduced to make the version tag change less often. """ -from rpython.rlib import jit, rerased +from rpython.rlib import jit, rerased, objectmodel from pypy.interpreter.baseobjspace import W_Root from pypy.objspace.std.dictmultiobject import ( @@ -162,8 +162,8 @@ def getitervalues(self, w_dict): return self.unerase(w_dict.dstorage).itervalues() - def getiteritems(self, w_dict): - return self.unerase(w_dict.dstorage).iteritems() + def getiteritems_with_hash(self, w_dict): + return objectmodel.iteritems_with_hash(self.unerase(w_dict.dstorage)) wrapkey = _wrapkey diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -511,7 +511,7 @@ def getitervalues(self, w_dict): raise NotImplementedError - def getiteritems(self, w_dict): + def getiteritems_with_hash(self, w_dict): raise NotImplementedError has_iterreversed = False @@ -634,7 +634,7 @@ def getitervalues(self, w_dict): return iter([]) - def getiteritems(self, w_dict): + def getiteritems_with_hash(self, w_dict): return iter([]) def getiterreversed(self, w_dict): @@ -751,11 +751,11 @@ class IterClassItems(BaseItemIterator): def __init__(self, space, strategy, impl): - self.iterator = strategy.getiteritems(impl) + self.iterator = strategy.getiteritems_with_hash(impl) BaseIteratorImplementation.__init__(self, space, strategy, impl) def next_item_entry(self): - for key, value in self.iterator: + for key, value, keyhash in self.iterator: return (wrapkey(self.space, key), wrapvalue(self.space, value)) else: @@ -793,10 +793,10 @@ # the logic is to call prepare_dict_update() after the first setitem(): # it gives the w_updatedict a chance to switch its strategy. if 1: # (preserve indentation) - iteritems = self.getiteritems(w_dict) + iteritemsh = self.getiteritems_with_hash(w_dict) if not same_strategy(self, w_updatedict): # Different strategy. Try to copy one item of w_dict - for key, value in iteritems: + for key, value, keyhash in iteritemsh: w_key = wrapkey(self.space, key) w_value = wrapvalue(self.space, value) w_updatedict.setitem(w_key, w_value) @@ -807,7 +807,7 @@ w_updatedict.strategy.prepare_update(w_updatedict, count) # If the strategy is still different, continue the slow way if not same_strategy(self, w_updatedict): - for key, value in iteritems: + for key, value, keyhash in iteritemsh: w_key = wrapkey(self.space, key) w_value = wrapvalue(self.space, value) w_updatedict.setitem(w_key, w_value) @@ -820,8 +820,8 @@ # wrapping/unwrapping the key. assert setitem_untyped is not None dstorage = w_updatedict.dstorage - for key, value in iteritems: - setitem_untyped(self, dstorage, key, value) + for key, value, keyhash in iteritemsh: + setitem_untyped(self, dstorage, key, value, keyhash) def same_strategy(self, w_otherdict): return (setitem_untyped is not None and @@ -945,8 +945,8 @@ def getitervalues(self, w_dict): return self.unerase(w_dict.dstorage).itervalues() - def getiteritems(self, w_dict): - return self.unerase(w_dict.dstorage).iteritems() + def getiteritems_with_hash(self, w_dict): + return objectmodel.iteritems_with_hash(self.unerase(w_dict.dstorage)) def getiterreversed(self, w_dict): return objectmodel.reversed_dict(self.unerase(w_dict.dstorage)) @@ -955,8 +955,9 @@ objectmodel.prepare_dict_update(self.unerase(w_dict.dstorage), num_extra) - def setitem_untyped(self, dstorage, key, w_value): - self.unerase(dstorage)[key] = w_value + def setitem_untyped(self, dstorage, key, w_value, keyhash): + d = self.unerase(dstorage) + objectmodel.setitem_with_hash(d, key, keyhash, w_value) class ObjectDictStrategy(AbstractTypedStrategy, DictStrategy): diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py --- a/pypy/objspace/std/dictproxyobject.py +++ b/pypy/objspace/std/dictproxyobject.py @@ -1,4 +1,5 @@ from rpython.rlib import rerased +from rpython.rlib.objectmodel import iteritems_with_hash from pypy.interpreter.error import OperationError, oefmt from pypy.objspace.std.dictmultiobject import ( @@ -103,8 +104,8 @@ return self.unerase(w_dict.dstorage).dict_w.iterkeys() def getitervalues(self, w_dict): return self.unerase(w_dict.dstorage).dict_w.itervalues() - def getiteritems(self, w_dict): - return self.unerase(w_dict.dstorage).dict_w.iteritems() + def getiteritems_with_hash(self, w_dict): + return iteritems_with_hash(self.unerase(w_dict.dstorage).dict_w) def wrapkey(space, key): return space.wrap(key) def wrapvalue(space, value): diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py --- a/pypy/objspace/std/kwargsdict.py +++ b/pypy/objspace/std/kwargsdict.py @@ -3,7 +3,7 @@ Based on two lists containing unwrapped key value pairs. """ -from rpython.rlib import jit, rerased +from rpython.rlib import jit, rerased, objectmodel from pypy.objspace.std.dictmultiobject import ( BytesDictStrategy, DictStrategy, EmptyDictStrategy, ObjectDictStrategy, @@ -165,13 +165,14 @@ def getitervalues(self, w_dict): return iter(self.unerase(w_dict.dstorage)[1]) - def getiteritems(self, w_dict): - return Zip(*self.unerase(w_dict.dstorage)) + def getiteritems_with_hash(self, w_dict): + keys, values_w = self.unerase(w_dict.dstorage) + return ZipItemsWithHash(keys, values_w) wrapkey = _wrapkey -class Zip(object): +class ZipItemsWithHash(object): def __init__(self, list1, list2): assert len(list1) == len(list2) self.list1 = list1 @@ -186,6 +187,7 @@ if i >= len(self.list1): raise StopIteration self.i = i + 1 - return (self.list1[i], self.list2[i]) + key = self.list1[i] + return (key, self.list2[i], objectmodel.compute_hash(key)) create_iterator_classes(KwargsDictStrategy) diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -8,6 +8,8 @@ from pypy.objspace.std.unicodeobject import W_UnicodeObject from rpython.rlib.objectmodel import r_dict +from rpython.rlib.objectmodel import iterkeys_with_hash, contains_with_hash +from rpython.rlib.objectmodel import setitem_with_hash, delitem_with_hash from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib import rerased, jit @@ -961,12 +963,12 @@ return self.erase(result_dict) def _difference_unwrapped(self, w_set, w_other): - iterator = self.unerase(w_set.sstorage).iterkeys() + self_dict = self.unerase(w_set.sstorage) other_dict = self.unerase(w_other.sstorage) result_dict = self.get_empty_dict() - for key in iterator: - if key not in other_dict: - result_dict[key] = None + for key, keyhash in iterkeys_with_hash(self_dict): + if not contains_with_hash(other_dict, key, keyhash): + setitem_with_hash(result_dict, key, keyhash, None) return self.erase(result_dict) def _difference_base(self, w_set, w_other): @@ -989,10 +991,10 @@ if w_set.sstorage is w_other.sstorage: my_dict.clear() return - iterator = self.unerase(w_other.sstorage).iterkeys() - for key in iterator: + other_dict = self.unerase(w_other.sstorage) + for key, keyhash in iterkeys_with_hash(other_dict): try: - del my_dict[key] + delitem_with_hash(my_dict, key, keyhash) except KeyError: pass @@ -1020,12 +1022,12 @@ d_new = self.get_empty_dict() d_this = self.unerase(w_set.sstorage) d_other = self.unerase(w_other.sstorage) - for key in d_other.keys(): - if not key in d_this: - d_new[key] = None - for key in d_this.keys(): - if not key in d_other: - d_new[key] = None + for key, keyhash in iterkeys_with_hash(d_other): + if not contains_with_hash(d_this, key, keyhash): + setitem_with_hash(d_new, key, keyhash, None) + for key, keyhash in iterkeys_with_hash(d_this): + if not contains_with_hash(d_other, key, keyhash): + setitem_with_hash(d_new, key, keyhash, None) storage = self.erase(d_new) return storage @@ -1105,9 +1107,9 @@ result = self.get_empty_dict() d_this = self.unerase(w_set.sstorage) d_other = self.unerase(w_other.sstorage) - for key in d_this: - if key in d_other: - result[key] = None + for key, keyhash in iterkeys_with_hash(d_this): + if contains_with_hash(d_other, key, keyhash): + setitem_with_hash(result, key, keyhash, None) return self.erase(result) def intersect(self, w_set, w_other): @@ -1125,9 +1127,10 @@ w_set.sstorage = storage def _issubset_unwrapped(self, w_set, w_other): + d_set = self.unerase(w_set.sstorage) d_other = self.unerase(w_other.sstorage) - for item in self.unerase(w_set.sstorage): - if not item in d_other: + for key, keyhash in iterkeys_with_hash(d_set): + if not contains_with_hash(d_other, key, keyhash): return False return True @@ -1152,8 +1155,8 @@ def _isdisjoint_unwrapped(self, w_set, w_other): d_set = self.unerase(w_set.sstorage) d_other = self.unerase(w_other.sstorage) - for key in d_set: - if key in d_other: + for key, keyhash in iterkeys_with_hash(d_set): + if contains_with_hash(d_other, key, keyhash): return False return True diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -520,26 +520,32 @@ return dic1.__class__(dic1.dictdef.union(dic2.dictdef)) +def _dict_can_only_throw_keyerror(s_dct, *ignore): + if s_dct.dictdef.dictkey.custom_eq_hash: + return None # r_dict: can throw anything + return [KeyError] + +def _dict_can_only_throw_nothing(s_dct, *ignore): + if s_dct.dictdef.dictkey.custom_eq_hash: + return None # r_dict: can throw anything + return [] # else: no possible exception + + class __extend__(pairtype(SomeDict, SomeObject)): - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - def getitem((dic1, obj2)): dic1.dictdef.generalize_key(obj2) return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw + getitem.can_only_throw = _dict_can_only_throw_keyerror def setitem((dic1, obj2), s_value): dic1.dictdef.generalize_key(obj2) dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw + setitem.can_only_throw = _dict_can_only_throw_nothing def delitem((dic1, obj2)): dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw + delitem.can_only_throw = _dict_can_only_throw_keyerror class __extend__(pairtype(SomeTuple, SomeInteger)): diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -4,6 +4,7 @@ from __future__ import absolute_import +from rpython.tool.pairtype import pair from rpython.flowspace.operation import op from rpython.flowspace.model import const, Constant from rpython.flowspace.argument import CallSpec @@ -11,11 +12,13 @@ SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, SomeFloat, SomeIterator, SomePBC, SomeNone, SomeType, s_ImpossibleValue, - s_Bool, s_None, unionof, add_knowntypedata, + s_Bool, s_None, s_Int, unionof, add_knowntypedata, HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray) from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue from rpython.annotator import builtin from rpython.annotator.binaryop import _clone ## XXX where to put this? +from rpython.annotator.binaryop import _dict_can_only_throw_keyerror +from rpython.annotator.binaryop import _dict_can_only_throw_nothing from rpython.annotator.model import AnnotatorError from rpython.annotator.argument import simple_args, complex_args @@ -364,20 +367,19 @@ raise AnnotatorError("%s: not proven to have non-negative stop" % error) -def _can_only_throw(s_dct, *ignore): - if s_dct.dictdef.dictkey.custom_eq_hash: - return None # r_dict: can throw anything - return [] # else: no possible exception - -@op.contains.register(SomeDict) -def contains_SomeDict(annotator, dct, element): - annotator.annotation(dct).dictdef.generalize_key(annotator.annotation(element)) - if annotator.annotation(dct)._is_empty(): +def dict_contains(s_dct, s_element): + s_dct.dictdef.generalize_key(s_element) + if s_dct._is_empty(): s_bool = SomeBool() s_bool.const = False return s_bool return s_Bool -contains_SomeDict.can_only_throw = _can_only_throw + +@op.contains.register(SomeDict) +def contains_SomeDict(annotator, dct, element): + return dict_contains(annotator.annotation(dct), + annotator.annotation(element)) +contains_SomeDict.can_only_throw = _dict_can_only_throw_nothing class __extend__(SomeDict): @@ -401,16 +403,22 @@ return self.dictdef.read_key() elif variant == 'values': return self.dictdef.read_value() - elif variant == 'items': + elif variant == 'items' or variant == 'items_with_hash': s_key = self.dictdef.read_key() s_value = self.dictdef.read_value() if (isinstance(s_key, SomeImpossibleValue) or isinstance(s_value, SomeImpossibleValue)): return s_ImpossibleValue - else: + elif variant == 'items': return SomeTuple((s_key, s_value)) - else: - raise ValueError + elif variant == 'items_with_hash': + return SomeTuple((s_key, s_value, s_Int)) + elif variant == 'keys_with_hash': + s_key = self.dictdef.read_key() + if isinstance(s_key, SomeImpossibleValue): + return s_ImpossibleValue + return SomeTuple((s_key, s_Int)) + raise ValueError(variant) def method_get(self, key, dfl): self.dictdef.generalize_key(key) @@ -448,6 +456,12 @@ def method_iteritems(self): return SomeIterator(self, 'items') + def method_iterkeys_with_hash(self): + return SomeIterator(self, 'keys_with_hash') + + def method_iteritems_with_hash(self): + return SomeIterator(self, 'items_with_hash') + def method_clear(self): pass @@ -460,6 +474,22 @@ self.dictdef.generalize_value(s_dfl) return self.dictdef.read_value() + def method_contains_with_hash(self, s_key, s_hash): + return dict_contains(self, s_key) + method_contains_with_hash.can_only_throw = _dict_can_only_throw_nothing + + def method_setitem_with_hash(self, s_key, s_hash, s_value): + pair(self, s_key).setitem(s_value) + method_setitem_with_hash.can_only_throw = _dict_can_only_throw_nothing + + def method_getitem_with_hash(self, s_key, s_hash): + return pair(self, s_key).getitem() + method_getitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror + + def method_delitem_with_hash(self, s_key, s_hash): + pair(self, s_key).delitem() + method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror + @op.contains.register(SomeString) @op.contains.register(SomeUnicodeString) def contains_String(annotator, string, char): diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py --- a/rpython/doc/conf.py +++ b/rpython/doc/conf.py @@ -68,7 +68,7 @@ # The short X.Y version. version = '2.6' # The full version, including alpha/beta/rc tags. -release = '2.6.0' +release = '2.6.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -114,10 +114,7 @@ def build_frame_realloc_slowpath(self): mc = codebuf.MachineCodeBlockWrapper() self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) - # this is the gcmap stored by push_gcmap(mov=True) in _check_stack_frame - mc.MOV_rs(ecx.value, WORD) - gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.MOV_br(gcmap_ofs, ecx.value) + # the caller already did push_gcmap(store=True) if IS_X86_64: mc.MOV_rs(esi.value, WORD*2) @@ -147,7 +144,7 @@ self._load_shadowstack_top_in_ebx(mc, gcrootmap) mc.MOV_mr((ebx.value, -WORD), eax.value) - mc.MOV_bi(gcmap_ofs, 0) + self.pop_gcmap(mc) # cancel the push_gcmap(store=True) in the caller self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) mc.RET() self._frame_realloc_slowpath = mc.materialize(self.cpu, []) @@ -164,6 +161,7 @@ # the end of this function. self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax], supports_floats, callee_only) + # the caller already did push_gcmap(store=True) if IS_X86_64: mc.SUB(esp, imm(WORD)) # alignment self.set_extra_stack_depth(mc, 2 * WORD) @@ -182,8 +180,8 @@ mc.ADD(esp, imm(WORD * 7)) self.set_extra_stack_depth(mc, 0) self._reload_frame_if_necessary(mc, align_stack=True) + self.pop_gcmap(mc) # cancel the push_gcmap(store=True) in the caller self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only) - self.pop_gcmap(mc) # push_gcmap(store=True) done by the caller mc.RET() return mc.materialize(self.cpu, []) @@ -203,10 +201,7 @@ assert kind in ['fixed', 'str', 'unicode', 'var'] mc = codebuf.MachineCodeBlockWrapper() self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats) - # store the gc pattern - ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.MOV_rs(ecx.value, WORD) - mc.MOV_br(ofs, ecx.value) + # the caller already did push_gcmap(store=True) # if kind == 'fixed': addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() @@ -258,8 +253,7 @@ self.set_extra_stack_depth(mc, 0) self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) mc.MOV(edi, heap(nursery_free_adr)) # load this in EDI - # clear the gc pattern - mc.MOV_bi(ofs, 0) + self.pop_gcmap(mc) # push_gcmap(store=True) done by the caller mc.RET() # # If the slowpath malloc failed, we raise a MemoryError that @@ -646,7 +640,7 @@ jg_location = mc.get_relative_pos() mc.MOV_si(WORD, 0xffffff) # force writing 32 bit ofs2 = mc.get_relative_pos() - 4 - self.push_gcmap(mc, gcmap, mov=True) + self.push_gcmap(mc, gcmap, store=True) mc.CALL(imm(self._frame_realloc_slowpath)) # patch the JG above offset = mc.get_relative_pos() - jg_location @@ -1797,12 +1791,9 @@ self.mc.JMP(imm(target)) return startpos - def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): + def push_gcmap(self, mc, gcmap, push=False, store=False): if push: mc.PUSH(imm(rffi.cast(lltype.Signed, gcmap))) - elif mov: - mc.MOV(RawEspLoc(0, REF), - imm(rffi.cast(lltype.Signed, gcmap))) else: assert store ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') @@ -2280,7 +2271,7 @@ self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr = self.mc.get_relative_pos() # save the gcmap - self.push_gcmap(self.mc, gcmap, mov=True) + self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 @@ -2301,7 +2292,7 @@ self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr = self.mc.get_relative_pos() # save the gcmap - self.push_gcmap(self.mc, gcmap, mov=True) + self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 @@ -2354,7 +2345,7 @@ assert 0 < offset <= 127 self.mc.overwrite(jmp_adr0-1, chr(offset)) # save the gcmap - self.push_gcmap(self.mc, gcmap, mov=True) # mov into RawEspLoc(0) + self.push_gcmap(self.mc, gcmap, store=True) if kind == rewrite.FLAG_ARRAY: self.mc.MOV_si(WORD, itemsize) self.mc.MOV(edi, lengthloc) diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -788,6 +788,71 @@ d = d.keys() return reversed(d) +def _expected_hash(d, key): + if isinstance(d, r_dict): + return d.key_hash(key) + else: + return compute_hash(key) + +def _iterkeys_with_hash_untranslated(d): + for k in d: + yield (k, _expected_hash(d, k)) + +@specialize.call_location() +def iterkeys_with_hash(d): + """Iterates (key, hash) pairs without recomputing the hash.""" + if not we_are_translated(): + return _iterkeys_with_hash_untranslated(d) + return d.iterkeys_with_hash() + +def _iteritems_with_hash_untranslated(d): + for k, v in d.iteritems(): + yield (k, v, _expected_hash(d, k)) + +@specialize.call_location() +def iteritems_with_hash(d): + """Iterates (key, value, keyhash) triples without recomputing the hash.""" + if not we_are_translated(): + return _iteritems_with_hash_untranslated(d) + return d.iteritems_with_hash() + +@specialize.call_location() +def contains_with_hash(d, key, h): + """Same as 'key in d'. The extra argument is the hash. Use this only + if you got the hash just now from some other ..._with_hash() function.""" + if not we_are_translated(): + assert _expected_hash(d, key) == h + return key in d + return d.contains_with_hash(key, h) + +@specialize.call_location() +def setitem_with_hash(d, key, h, value): + """Same as 'd[key] = value'. The extra argument is the hash. Use this only + if you got the hash just now from some other ..._with_hash() function.""" + if not we_are_translated(): + assert _expected_hash(d, key) == h + d[key] = value + return + d.setitem_with_hash(key, h, value) + +@specialize.call_location() +def getitem_with_hash(d, key, h): + """Same as 'd[key]'. The extra argument is the hash. Use this only + if you got the hash just now from some other ..._with_hash() function.""" + if not we_are_translated(): + assert _expected_hash(d, key) == h + return d[key] + return d.getitem_with_hash(key, h) + +@specialize.call_location() +def delitem_with_hash(d, key, h): + """Same as 'del d[key]'. The extra argument is the hash. Use this only + if you got the hash just now from some other ..._with_hash() function.""" + if not we_are_translated(): + assert _expected_hash(d, key) == h + del d[key] + return + d.delitem_with_hash(key, h) # ____________________________________________________________ diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -592,6 +592,97 @@ r = interpret(f, [29]) assert r == 1 +def test_iterkeys_with_hash(): + def f(i): + d = {i+.0: 5, i+.5: 6} + total = 0 + for k, h in iterkeys_with_hash(d): + total += k * h + total -= (i + 0.0) * compute_hash(i + 0.0) + total -= (i + 0.5) * compute_hash(i + 0.5) + return total + + assert f(29) == 0.0 + r = interpret(f, [29]) + assert r == 0.0 + +def test_iteritems_with_hash(): + def f(i): + d = {i+.0: 5, i+.5: 6} + total = 0 + for k, v, h in iteritems_with_hash(d): + total += k * h * v + total -= (i + 0.0) * compute_hash(i + 0.0) * 5 + total -= (i + 0.5) * compute_hash(i + 0.5) * 6 + return total + + assert f(29) == 0.0 + r = interpret(f, [29]) + assert r == 0.0 + +def test_contains_with_hash(): + def f(i): + d = {i+.5: 5} + assert contains_with_hash(d, i+.5, compute_hash(i+.5)) + assert not contains_with_hash(d, i+.3, compute_hash(i+.3)) + return 0 + + f(29) + interpret(f, [29]) + +def test_setitem_with_hash(): + def f(i): + d = {} + setitem_with_hash(d, i+.5, compute_hash(i+.5), 42) + setitem_with_hash(d, i+.6, compute_hash(i+.6), -612) + return d[i+.5] + + assert f(29) == 42 + res = interpret(f, [27]) + assert res == 42 + +def test_getitem_with_hash(): + def f(i): + d = {i+.5: 42, i+.6: -612} + return getitem_with_hash(d, i+.5, compute_hash(i+.5)) + + assert f(29) == 42 + res = interpret(f, [27]) + assert res == 42 + +def test_delitem_with_hash(): + def f(i): + d = {i+.5: 42, i+.6: -612} + delitem_with_hash(d, i+.5, compute_hash(i+.5)) + try: + delitem_with_hash(d, i+.5, compute_hash(i+.5)) + except KeyError: + pass + else: + raise AssertionError + return 0 + + f(29) + interpret(f, [27]) + +def test_rdict_with_hash(): + def f(i): + d = r_dict(strange_key_eq, strange_key_hash) + h = strange_key_hash("abc") + assert h == strange_key_hash("aXX") and strange_key_eq("abc", "aXX") + setitem_with_hash(d, "abc", h, i) + assert getitem_with_hash(d, "aXX", h) == i + try: + getitem_with_hash(d, "bYY", strange_key_hash("bYY")) + except KeyError: + pass + else: + raise AssertionError + return 0 + + assert f(29) == 0 + interpret(f, [27]) + def test_import_from_mixin(): class M: # old-style def f(self): pass diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py --- a/rpython/rtyper/lltypesystem/rordereddict.py +++ b/rpython/rtyper/lltypesystem/rordereddict.py @@ -335,6 +335,14 @@ hop.exception_cannot_occur() return DictIteratorRepr(self, "items").newiter(hop) + def rtype_method_iterkeys_with_hash(self, hop): + hop.exception_cannot_occur() + return DictIteratorRepr(self, "keys_with_hash").newiter(hop) + + def rtype_method_iteritems_with_hash(self, hop): + hop.exception_cannot_occur() + return DictIteratorRepr(self, "items_with_hash").newiter(hop) + def rtype_method_clear(self, hop): v_dict, = hop.inputargs(self) hop.exception_cannot_occur() @@ -358,6 +366,41 @@ v_res = hop.gendirectcall(target, *v_args) return self.recast_value(hop.llops, v_res) + def rtype_method_contains_with_hash(self, hop): + v_dict, v_key, v_hash = hop.inputargs(self, self.key_repr, + lltype.Signed) + hop.exception_is_here() + return hop.gendirectcall(ll_dict_contains_with_hash, + v_dict, v_key, v_hash) + + def rtype_method_setitem_with_hash(self, hop): + v_dict, v_key, v_hash, v_value = hop.inputargs( + self, self.key_repr, lltype.Signed, self.value_repr) + if self.custom_eq_hash: + hop.exception_is_here() + else: + hop.exception_cannot_occur() + hop.gendirectcall(ll_dict_setitem_with_hash, + v_dict, v_key, v_hash, v_value) + + def rtype_method_getitem_with_hash(self, hop): + v_dict, v_key, v_hash = hop.inputargs( + self, self.key_repr, lltype.Signed) + if not self.custom_eq_hash: + hop.has_implicit_exception(KeyError) # record that we know about it + hop.exception_is_here() + v_res = hop.gendirectcall(ll_dict_getitem_with_hash, + v_dict, v_key, v_hash) + return self.recast_value(hop.llops, v_res) + + def rtype_method_delitem_with_hash(self, hop): + v_dict, v_key, v_hash = hop.inputargs( + self, self.key_repr, lltype.Signed) + if not self.custom_eq_hash: + hop.has_implicit_exception(KeyError) # record that we know about it + hop.exception_is_here() + hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash) + class __extend__(pairtype(OrderedDictRepr, rmodel.Repr)): def rtype_getitem((r_dict, r_key), hop): @@ -373,7 +416,7 @@ if not r_dict.custom_eq_hash: hop.has_implicit_exception(KeyError) # record that we know about it hop.exception_is_here() - return hop.gendirectcall(ll_dict_delitem, v_dict, v_key) + hop.gendirectcall(ll_dict_delitem, v_dict, v_key) def rtype_setitem((r_dict, r_key), hop): v_dict, v_key, v_value = hop.inputargs(r_dict, r_dict.key_repr, r_dict.value_repr) @@ -541,16 +584,21 @@ return bool(d) and d.num_live_items != 0 def ll_dict_getitem(d, key): - index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP) + return ll_dict_getitem_with_hash(d, key, d.keyhash(key)) + +def ll_dict_getitem_with_hash(d, key, hash): + index = d.lookup_function(d, key, hash, FLAG_LOOKUP) if index >= 0: return d.entries[index].value else: raise KeyError def ll_dict_setitem(d, key, value): - hash = d.keyhash(key) + ll_dict_setitem_with_hash(d, key, d.keyhash(key), value) + +def ll_dict_setitem_with_hash(d, key, hash, value): index = d.lookup_function(d, key, hash, FLAG_STORE) - return _ll_dict_setitem_lookup_done(d, key, value, hash, index) + _ll_dict_setitem_lookup_done(d, key, value, hash, index) # It may be safe to look inside always, it has a few branches though, and their # frequencies needs to be investigated. @@ -731,7 +779,10 @@ def ll_dict_delitem(d, key): - index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE) + ll_dict_delitem_with_hash(d, key, d.keyhash(key)) + +def ll_dict_delitem_with_hash(d, key, hash): + index = d.lookup_function(d, key, hash, FLAG_DELETE) if index < 0: raise KeyError _ll_dict_del(d, index) @@ -1214,7 +1265,10 @@ ll_dict_items = _make_ll_keys_values_items('items') def ll_dict_contains(d, key): - i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP) + return ll_dict_contains_with_hash(d, key, d.keyhash(key)) + +def ll_dict_contains_with_hash(d, key, hash): + i = d.lookup_function(d, key, hash, FLAG_LOOKUP) return i >= 0 def _ll_getnextitem(dic): diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py --- a/rpython/rtyper/rdict.py +++ b/rpython/rtyper/rdict.py @@ -72,20 +72,14 @@ return hop.gendirectcall(self.ll_dictiter, citerptr, v_dict) def rtype_next(self, hop): - variant = self.variant v_iter, = hop.inputargs(self) # record that we know about these two possible exceptions hop.has_implicit_exception(StopIteration) hop.has_implicit_exception(RuntimeError) hop.exception_is_here() v_index = hop.gendirectcall(self._ll_dictnext, v_iter) - if variant == 'items' and hop.r_result.lowleveltype != lltype.Void: - # this allocates the tuple for the result, directly in the function - # where it will be used (likely). This will let it be removed. - c1 = hop.inputconst(lltype.Void, hop.r_result.lowleveltype.TO) - cflags = hop.inputconst(lltype.Void, {'flavor': 'gc'}) - v_result = hop.genop('malloc', [c1, cflags], - resulttype = hop.r_result.lowleveltype) + # + # read 'iter.dict.entries' DICT = self.lowleveltype.TO.dict c_dict = hop.inputconst(lltype.Void, 'dict') v_dict = hop.genop('getfield', [v_iter, c_dict], resulttype=DICT) @@ -93,32 +87,61 @@ c_entries = hop.inputconst(lltype.Void, 'entries') v_entries = hop.genop('getfield', [v_dict, c_entries], resulttype=ENTRIES) - if variant != 'values': - KEY = ENTRIES.TO.OF.key - c_key = hop.inputconst(lltype.Void, 'key') - v_key = hop.genop('getinteriorfield', [v_entries, v_index, c_key], - resulttype=KEY) - if variant != 'keys' and variant != 'reversed': - VALUE = ENTRIES.TO.OF.value - c_value = hop.inputconst(lltype.Void, 'value') - v_value = hop.genop('getinteriorfield', [v_entries,v_index,c_value], - resulttype=VALUE) - if variant == 'keys' or variant == 'reversed': - return self.r_dict.recast_key(hop.llops, v_key) - elif variant == 'values': - return self.r_dict.recast_value(hop.llops, v_value) - elif hop.r_result.lowleveltype == lltype.Void: + # call the correct variant_*() method + method = getattr(self, 'variant_' + self.variant) + return method(hop, ENTRIES, v_entries, v_index) + + def get_tuple_result(self, hop, items_v): + # this allocates the tuple for the result, directly in the function + # where it will be used (likely). This will let it be removed. + if hop.r_result.lowleveltype is lltype.Void: return hop.inputconst(lltype.Void, None) - else: - assert variant == 'items' - ITEM0 = v_result.concretetype.TO.item0 - ITEM1 = v_result.concretetype.TO.item1 - if ITEM0 != v_key.concretetype: - v_key = hop.genop('cast_pointer', [v_key], resulttype=ITEM0) - if ITEM1 != v_value.concretetype: - v_value = hop.genop('cast_pointer', [v_value], resulttype=ITEM1) - c_item0 = hop.inputconst(lltype.Void, 'item0') - c_item1 = hop.inputconst(lltype.Void, 'item1') - hop.genop('setfield', [v_result, c_item0, v_key]) - hop.genop('setfield', [v_result, c_item1, v_value]) - return v_result + c1 = hop.inputconst(lltype.Void, hop.r_result.lowleveltype.TO) + cflags = hop.inputconst(lltype.Void, {'flavor': 'gc'}) + v_result = hop.genop('malloc', [c1, cflags], + resulttype = hop.r_result.lowleveltype) + for i, v_item in enumerate(items_v): + ITEM = getattr(v_result.concretetype.TO, 'item%d' % i) + if ITEM != v_item.concretetype: + assert isinstance(ITEM, lltype.Ptr) + v_item = hop.genop('cast_pointer', [v_item], resulttype=ITEM) + c_item = hop.inputconst(lltype.Void, 'item%d' % i) + hop.genop('setfield', [v_result, c_item, v_item]) + return v_result + + def variant_keys(self, hop, ENTRIES, v_entries, v_index): + KEY = ENTRIES.TO.OF.key + c_key = hop.inputconst(lltype.Void, 'key') + v_key = hop.genop('getinteriorfield', [v_entries, v_index, c_key], + resulttype=KEY) + return self.r_dict.recast_key(hop.llops, v_key) + + variant_reversed = variant_keys + + def variant_values(self, hop, ENTRIES, v_entries, v_index): + VALUE = ENTRIES.TO.OF.value + c_value = hop.inputconst(lltype.Void, 'value') + v_value = hop.genop('getinteriorfield', [v_entries,v_index,c_value], + resulttype=VALUE) + return self.r_dict.recast_value(hop.llops, v_value) + + def variant_items(self, hop, ENTRIES, v_entries, v_index): + v_key = self.variant_keys(hop, ENTRIES, v_entries, v_index) + v_value = self.variant_values(hop, ENTRIES, v_entries, v_index) + return self.get_tuple_result(hop, (v_key, v_value)) + + def variant_hashes(self, hop, ENTRIES, v_entries, v_index): + # there is not really a variant 'hashes', but this method is + # convenient for the following variants + return hop.gendirectcall(ENTRIES.TO.hash, v_entries, v_index) + + def variant_keys_with_hash(self, hop, ENTRIES, v_entries, v_index): + v_key = self.variant_keys(hop, ENTRIES, v_entries, v_index) + v_hash = self.variant_hashes(hop, ENTRIES, v_entries, v_index) + return self.get_tuple_result(hop, (v_key, v_hash)) + + def variant_items_with_hash(self, hop, ENTRIES, v_entries, v_index): + v_key = self.variant_keys(hop, ENTRIES, v_entries, v_index) + v_value = self.variant_values(hop, ENTRIES, v_entries, v_index) + v_hash = self.variant_hashes(hop, ENTRIES, v_entries, v_index) + return self.get_tuple_result(hop, (v_key, v_value, v_hash)) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit