Author: Matti Picus <[email protected]>
Branch: cpyext-for-merge
Changeset: r83967:05bf6936b917
Date: 2016-04-27 11:08 +0300
http://bitbucket.org/pypy/pypy/changeset/05bf6936b917/
Log: merge default into branch
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,9 @@
Remove a number of options from the build process that were never tested and
never set. Fix a performance bug in the method cache.
+
+.. branch: bitstring
+
+JIT: use bitstrings to compress the lists of read or written descrs
+that we attach to EffectInfo. Fixes a problem we had in
+remove-objspace-options.
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -162,7 +162,10 @@
# When translating PyPy, freeze the file name
# <builtin>/lastdirname/basename.py
# instead of freezing the complete translation-time path.
- filename = self.co_filename.lstrip('<').rstrip('>')
+ filename = self.co_filename
+ if filename.startswith('<builtin>'):
+ return
+ filename = filename.lstrip('<').rstrip('>')
if filename.lower().endswith('.pyc'):
filename = filename[:-1]
basename = os.path.basename(filename)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -742,7 +742,7 @@
def RAISE_VARARGS(self, nbargs, next_instr):
space = self.space
if nbargs == 0:
- last_operr = self._exc_info_unroll(space)
+ last_operr = self._exc_info_unroll(space, for_hidden=True)
if last_operr is None:
raise oefmt(space.w_TypeError,
"No active exception to reraise")
diff --git a/pypy/module/__pypy__/test/test_special.py
b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -60,6 +60,7 @@
import __pypy__
import sys
+ result = [False]
@__pypy__.hidden_applevel
def test_hidden_with_tb():
def not_hidden(): 1/0
@@ -68,9 +69,11 @@
assert sys.exc_info() == (None, None, None)
tb = __pypy__.get_hidden_tb()
assert tb.tb_frame.f_code.co_name == 'not_hidden'
- return True
+ result[0] = True
+ raise
else: return False
- assert test_hidden_with_tb()
+ raises(ZeroDivisionError, test_hidden_with_tb)
+ assert result[0]
def test_lookup_special(self):
from __pypy__ import lookup_special
diff --git a/pypy/module/cpyext/include/listobject.h
b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,2 +1,1 @@
#define PyList_GET_ITEM PyList_GetItem
-#define PyList_SET_ITEM PyList_SetItem
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -3,7 +3,7 @@
from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
build_type_checkers)
from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.module.cpyext.pyobject import Py_DecRef, PyObject
+from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, make_ref
from pypy.objspace.std.listobject import W_ListObject
from pypy.interpreter.error import OperationError
@@ -21,6 +21,25 @@
"""
return space.newlist([None] * len)
+@cpython_api([PyObject, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
+ result_borrowed=True)
+def PyList_SET_ITEM(space, w_list, index, w_item):
+ """Macro form of PyList_SetItem() without error checking. This is normally
+ only used to fill in new lists where there is no previous content.
+
+ This function "steals" a reference to item, and, unlike PyList_SetItem(),
+ does not discard a reference to any item that it being replaced; any
+ reference in list at position i will be leaked.
+ """
+ assert isinstance(w_list, W_ListObject)
+ assert 0 <= index < w_list.length()
+ # Deliberately leak, so that it can be safely decref'd.
+ make_ref(space, w_list.getitem(index))
+ Py_DecRef(space, w_item)
+ w_list.setitem(index, w_item)
+ return w_item
+
+
@cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
def PyList_SetItem(space, w_list, index, w_item):
"""Set the item at index index in list to item. Return 0 on success
diff --git a/pypy/module/cpyext/test/test_listobject.py
b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -136,3 +136,45 @@
l = [1, 2, 3]
module.setlistitem(l,0)
assert l == [None, 2, 3]
+
+ def test_get_item_macro(self):
+ module = self.import_extension('foo', [
+ ("test_get_item", "METH_NOARGS",
+ """
+ PyObject* o = PyList_New(1);
+
+ PyObject* o2 = PyInt_FromLong(0);
+ PyList_SET_ITEM(o, 0, o2);
+ o2 = NULL;
+
+ PyObject* o3 = PyList_GET_ITEM(o, 0);
+ Py_INCREF(o3);
+ Py_CLEAR(o);
+ return o3;
+ """)])
+ assert module.test_get_item() == 0
+
+ def test_set_item_macro(self):
+ """PyList_SET_ITEM leaks a reference to the target."""
+ module = self.import_extension('foo', [
+ ("test_refcount_diff_after_setitem", "METH_NOARGS",
+ """
+ PyObject* o = PyList_New(0);
+ PyObject* o2 = PyList_New(0);
+
+ PyList_Append(o, o2); // does not steal o2
+
+ Py_ssize_t refcount = Py_REFCNT(o2);
+
+ // Steal a reference to o2, but leak the old reference to o2.
+ // The net result should be no change in refcount.
+ PyList_SET_ITEM(o, 0, o2);
+
+ Py_ssize_t new_refcount = Py_REFCNT(o2);
+
+ Py_CLEAR(o);
+ Py_DECREF(o2); // append incref'd.
+ // Py_CLEAR(o2); // naive implementation would fail here.
+ return PyLong_FromSsize_t(new_refcount - refcount);
+ """)])
+ assert module.test_refcount_diff_after_setitem() == 0
diff --git a/rpython/annotator/test/test_annrpython.py
b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4577,6 +4577,13 @@
with py.test.raises(AnnotatorError):
a.build_types(f, [float])
+ def test_Ellipsis_not_rpython(self):
+ def f():
+ return Ellipsis
+ a = self.RPythonAnnotator()
+ e = py.test.raises(Exception, a.build_types, f, [])
+ assert str(e.value) == "Don't know how to represent Ellipsis"
+
def g(n):
return [0, 1, 2, n]
diff --git a/rpython/jit/backend/llgraph/runner.py
b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -479,6 +479,9 @@
all_descrs.append(v)
return all_descrs
+ def fetch_all_descrs(self):
+ return self.descrs.values()
+
def calldescrof(self, FUNC, ARGS, RESULT, effect_info):
key = ('call', getkind(RESULT),
tuple([getkind(A) for A in ARGS]),
diff --git a/rpython/jit/codewriter/effectinfo.py
b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -1,7 +1,9 @@
+import sys
from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem
from rpython.rtyper.rclass import OBJECT
from rpython.rtyper.lltypesystem import lltype, llmemory
from rpython.translator.backendopt.graphanalyze import BoolGraphAnalyzer
+from rpython.tool.algo import bitstring
class EffectInfo(object):
@@ -110,12 +112,20 @@
can_invalidate=False,
call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
extradescrs=None):
- key = (frozenset_or_none(readonly_descrs_fields),
- frozenset_or_none(readonly_descrs_arrays),
- frozenset_or_none(readonly_descrs_interiorfields),
- frozenset_or_none(write_descrs_fields),
- frozenset_or_none(write_descrs_arrays),
- frozenset_or_none(write_descrs_interiorfields),
+ readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
+ readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
+ readonly_descrs_interiorfields = frozenset_or_none(
+ readonly_descrs_interiorfields)
+ write_descrs_fields = frozenset_or_none(write_descrs_fields)
+ write_descrs_arrays = frozenset_or_none(write_descrs_arrays)
+ write_descrs_interiorfields = frozenset_or_none(
+ write_descrs_interiorfields)
+ key = (readonly_descrs_fields,
+ readonly_descrs_arrays,
+ readonly_descrs_interiorfields,
+ write_descrs_fields,
+ write_descrs_arrays,
+ write_descrs_interiorfields,
extraeffect,
oopspecindex,
can_invalidate)
@@ -139,22 +149,34 @@
assert write_descrs_arrays is not None
assert write_descrs_interiorfields is not None
result = object.__new__(cls)
- result.readonly_descrs_fields = readonly_descrs_fields
- result.readonly_descrs_arrays = readonly_descrs_arrays
- result.readonly_descrs_interiorfields = readonly_descrs_interiorfields
+ # the frozensets "._readonly_xxx" and "._write_xxx" should not be
+ # translated.
+ result._readonly_descrs_fields = readonly_descrs_fields
+ result._readonly_descrs_arrays = readonly_descrs_arrays
+ result._readonly_descrs_interiorfields = readonly_descrs_interiorfields
if extraeffect == EffectInfo.EF_LOOPINVARIANT or \
extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \
extraeffect == EffectInfo.EF_ELIDABLE_OR_MEMORYERROR or \
extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE:
# Ignore the writes. Note that this ignores also writes with
# no corresponding reads (rarely the case, but possible).
- result.write_descrs_fields = []
- result.write_descrs_arrays = []
- result.write_descrs_interiorfields = []
+ result._write_descrs_fields = frozenset()
+ result._write_descrs_arrays = frozenset()
+ result._write_descrs_interiorfields = frozenset()
else:
- result.write_descrs_fields = write_descrs_fields
- result.write_descrs_arrays = write_descrs_arrays
- result.write_descrs_interiorfields = write_descrs_interiorfields
+ result._write_descrs_fields = write_descrs_fields
+ result._write_descrs_arrays = write_descrs_arrays
+ result._write_descrs_interiorfields = write_descrs_interiorfields
+ # initialized later, in compute_bitstrings()
+ # (the goal of this is to make sure we don't build new EffectInfo
+ # instances after compute_bitstrings() is called)
+ result.bitstring_readonly_descrs_fields = Ellipsis
+ result.bitstring_readonly_descrs_arrays = Ellipsis
+ result.bitstring_readonly_descrs_interiorfields = Ellipsis
+ result.bitstring_write_descrs_fields = Ellipsis
+ result.bitstring_write_descrs_arrays = Ellipsis
+ result.bitstring_write_descrs_interiorfields = Ellipsis
+ #
result.extraeffect = extraeffect
result.can_invalidate = can_invalidate
result.oopspecindex = oopspecindex
@@ -162,9 +184,38 @@
result.call_release_gil_target = call_release_gil_target
if result.check_can_raise(ignore_memoryerror=True):
assert oopspecindex in cls._OS_CANRAISE
+
+ if (result._write_descrs_arrays is not None and
+ len(result._write_descrs_arrays) == 1):
+ # this is used only for ARRAYCOPY operations
+ [result.single_write_descr_array] = result._write_descrs_arrays
+ else:
+ result.single_write_descr_array = None
+
cls._cache[key] = result
return result
+ def check_readonly_descr_field(self, fielddescr):
+ return bitstring.bitcheck(self.bitstring_readonly_descrs_fields,
+ fielddescr.ei_index)
+ def check_write_descr_field(self, fielddescr):
+ return bitstring.bitcheck(self.bitstring_write_descrs_fields,
+ fielddescr.ei_index)
+ def check_readonly_descr_array(self, arraydescr):
+ return bitstring.bitcheck(self.bitstring_readonly_descrs_arrays,
+ arraydescr.ei_index)
+ def check_write_descr_array(self, arraydescr):
+ return bitstring.bitcheck(self.bitstring_write_descrs_arrays,
+ arraydescr.ei_index)
+ def check_readonly_descr_interiorfield(self, interiorfielddescr):
+ # NOTE: this is not used so far
+ return
bitstring.bitcheck(self.bitstring_readonly_descrs_interiorfields,
+ interiorfielddescr.ei_index)
+ def check_write_descr_interiorfield(self, interiorfielddescr):
+ # NOTE: this is not used so far
+ return bitstring.bitcheck(self.bitstring_write_descrs_interiorfields,
+ interiorfielddescr.ei_index)
+
def check_can_raise(self, ignore_memoryerror=False):
if ignore_memoryerror:
return self.extraeffect > self.EF_ELIDABLE_OR_MEMORYERROR
@@ -382,3 +433,88 @@
assert funcptr
return funcptr
funcptr_for_oopspec._annspecialcase_ = 'specialize:arg(1)'
+
+# ____________________________________________________________
+
+def compute_bitstrings(all_descrs):
+ # Compute the bitstrings in the EffectInfo,
+ # bitstring_{readonly,write}_descrs_{fieldd,arrays,interiordescrs},
+ # and for each FieldDescrs and ArrayDescrs compute 'ei_index'.
+ # Each bit in the bitstrings says whether this Descr is present in
+ # this EffectInfo or not. We try to share the value of 'ei_index'
+ # across multiple Descrs if they always give the same answer (in
+ # PyPy, it reduces the length of the bitstrings from 4000+ to
+ # 373).
+ from rpython.jit.codewriter.policy import log
+
+ log("compute_bitstrings:")
+ effectinfos = []
+ descrs = {'fields': set(), 'arrays': set(), 'interiorfields': set()}
+ for descr in all_descrs:
+ if hasattr(descr, 'get_extra_info'):
+ ei = descr.get_extra_info()
+ if ei is None:
+ continue
+ if ei._readonly_descrs_fields is None:
+ for key in descrs:
+ assert getattr(ei, '_readonly_descrs_' + key) is None
+ assert getattr(ei, '_write_descrs_' + key) is None
+ setattr(ei, 'bitstring_readonly_descrs_' + key, None)
+ setattr(ei, 'bitstring_write_descrs_' + key, None)
+ else:
+ effectinfos.append(ei)
+ for key in descrs:
+ descrs[key].update(getattr(ei, '_readonly_descrs_' + key))
+ descrs[key].update(getattr(ei, '_write_descrs_' + key))
+ else:
+ descr.ei_index = sys.maxint
+ log(" %d effectinfos:" % (len(effectinfos),))
+ for key in sorted(descrs):
+ log(" %d descrs for %s" % (len(descrs[key]), key))
+
+ seen = set()
+ for key in descrs:
+ all_sets = []
+ for descr in descrs[key]:
+ eisetr = [ei for ei in effectinfos
+ if descr in getattr(ei, '_readonly_descrs_' + key)]
+ eisetw = [ei for ei in effectinfos
+ if descr in getattr(ei, '_write_descrs_' + key)]
+ # these are the set of all ei such that this descr is in
+ # ei._readonly_descrs or ei._write_descrs
+ eisetr = frozenset(eisetr)
+ eisetw = frozenset(eisetw)
+ all_sets.append((descr, eisetr, eisetw))
+
+ # heuristic to reduce the total size of the bitstrings: start with
+ # numbering the descrs that are seen in many EffectInfos. If instead,
+ # by lack of chance, such a descr had a high number, then all these
+ # EffectInfos' bitstrings would need to store the same high number.
+ def size_of_both_sets((d, r, w)):
+ return len(r) + len(w)
+ all_sets.sort(key=size_of_both_sets, reverse=True)
+
+ mapping = {}
+ for (descr, eisetr, eisetw) in all_sets:
+ assert descr.ei_index == sys.maxint # not modified yet
+ descr.ei_index = mapping.setdefault((eisetr, eisetw), len(mapping))
+
+ for ei in effectinfos:
+ bitstrr = [descr.ei_index
+ for descr in getattr(ei, '_readonly_descrs_' + key)]
+ bitstrw = [descr.ei_index
+ for descr in getattr(ei, '_write_descrs_' + key)]
+ assert sys.maxint not in bitstrr
+ assert sys.maxint not in bitstrw
+ bitstrr = bitstring.make_bitstring(bitstrr)
+ bitstrw = bitstring.make_bitstring(bitstrw)
+ setattr(ei, 'bitstring_readonly_descrs_' + key, bitstrr)
+ setattr(ei, 'bitstring_write_descrs_' + key, bitstrw)
+ seen.add(bitstrr)
+ seen.add(bitstrw)
+
+ if seen:
+ mean_length = float(sum(len(x) for x in seen)) / len(seen)
+ max_length = max(len(x) for x in seen)
+ log("-> %d bitstrings, mean length %.1f, max length %d" % (
+ len(seen), mean_length, max_length))
diff --git a/rpython/jit/codewriter/test/test_effectinfo.py
b/rpython/jit/codewriter/test/test_effectinfo.py
--- a/rpython/jit/codewriter/test/test_effectinfo.py
+++ b/rpython/jit/codewriter/test/test_effectinfo.py
@@ -1,11 +1,12 @@
-import pytest
+import pytest, sys
from rpython.jit.codewriter.effectinfo import (effectinfo_from_writeanalyze,
- EffectInfo, VirtualizableAnalyzer)
+ EffectInfo, VirtualizableAnalyzer, compute_bitstrings)
from rpython.rlib import jit
from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper.rclass import OBJECT
from rpython.translator.translator import TranslationContext, graphof
+from rpython.tool.algo.bitstring import bitcheck
class FakeCPU(object):
@@ -29,37 +30,39 @@
S = lltype.GcStruct("S", ("a", lltype.Signed))
effects = frozenset([("readstruct", lltype.Ptr(S), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert list(effectinfo.readonly_descrs_fields) == [('fielddescr', S, "a")]
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert list(effectinfo._readonly_descrs_fields) == [('fielddescr', S, "a")]
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
+ assert effectinfo.single_write_descr_array is None
def test_include_write_field():
S = lltype.GcStruct("S", ("a", lltype.Signed))
effects = frozenset([("struct", lltype.Ptr(S), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")]
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert list(effectinfo._write_descrs_fields) == [('fielddescr', S, "a")]
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_include_read_array():
A = lltype.GcArray(lltype.Signed)
effects = frozenset([("readarray", lltype.Ptr(A))])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert list(effectinfo.readonly_descrs_arrays) == [('arraydescr', A)]
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert list(effectinfo._readonly_descrs_arrays) == [('arraydescr', A)]
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_include_write_array():
A = lltype.GcArray(lltype.Signed)
effects = frozenset([("array", lltype.Ptr(A))])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)]
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert list(effectinfo._write_descrs_arrays) == [('arraydescr', A)]
+ assert effectinfo.single_write_descr_array == ('arraydescr', A)
def test_dont_include_read_and_write_field():
@@ -67,9 +70,9 @@
effects = frozenset([("readstruct", lltype.Ptr(S), "a"),
("struct", lltype.Ptr(S), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert list(effectinfo.write_descrs_fields) == [('fielddescr', S, "a")]
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert list(effectinfo._write_descrs_fields) == [('fielddescr', S, "a")]
+ assert not effectinfo._write_descrs_arrays
def test_dont_include_read_and_write_array():
@@ -77,34 +80,34 @@
effects = frozenset([("readarray", lltype.Ptr(A)),
("array", lltype.Ptr(A))])
effectinfo = effectinfo_from_writeanalyze(effects, FakeCPU())
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.readonly_descrs_arrays
- assert not effectinfo.write_descrs_fields
- assert list(effectinfo.write_descrs_arrays) == [('arraydescr', A)]
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._readonly_descrs_arrays
+ assert not effectinfo._write_descrs_fields
+ assert list(effectinfo._write_descrs_arrays) == [('arraydescr', A)]
def test_filter_out_typeptr():
effects = frozenset([("struct", lltype.Ptr(OBJECT), "typeptr")])
effectinfo = effectinfo_from_writeanalyze(effects, None)
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_filter_out_array_of_void():
effects = frozenset([("array", lltype.Ptr(lltype.GcArray(lltype.Void)))])
effectinfo = effectinfo_from_writeanalyze(effects, None)
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
def test_filter_out_struct_with_void():
effects = frozenset([("struct", lltype.Ptr(lltype.GcStruct("x", ("a",
lltype.Void))), "a")])
effectinfo = effectinfo_from_writeanalyze(effects, None)
- assert not effectinfo.readonly_descrs_fields
- assert not effectinfo.write_descrs_fields
- assert not effectinfo.write_descrs_arrays
+ assert not effectinfo._readonly_descrs_fields
+ assert not effectinfo._write_descrs_fields
+ assert not effectinfo._write_descrs_arrays
class TestVirtualizableAnalyzer(object):
@@ -138,3 +141,64 @@
res = self.analyze(entry, [int])
assert not res
+
+
+def test_compute_bitstrings():
+ class FDescr:
+ pass
+ class ADescr:
+ pass
+ class CDescr:
+ def __init__(self, ei):
+ self._ei = ei
+ def get_extra_info(self):
+ return self._ei
+
+ f1descr = FDescr()
+ f2descr = FDescr()
+ f3descr = FDescr()
+ a1descr = ADescr()
+ a2descr = ADescr()
+
+ ei1 = EffectInfo(None, None, None, None, None, None,
+ EffectInfo.EF_RANDOM_EFFECTS)
+ ei2 = EffectInfo([f1descr], [], [], [], [], [])
+ ei3 = EffectInfo([f1descr], [a1descr, a2descr], [], [f2descr], [], [])
+
+ compute_bitstrings([CDescr(ei1), CDescr(ei2), CDescr(ei3),
+ f1descr, f2descr, f3descr, a1descr, a2descr])
+
+ assert f1descr.ei_index in (0, 1)
+ assert f2descr.ei_index == 1 - f1descr.ei_index
+ assert f3descr.ei_index == sys.maxint
+ assert a1descr.ei_index == 0
+ assert a2descr.ei_index == 0
+
+ assert ei1.bitstring_readonly_descrs_fields is None
+ assert ei1.bitstring_readonly_descrs_arrays is None
+ assert ei1.bitstring_write_descrs_fields is None
+
+ def expand(bitstr):
+ return [n for n in range(10) if bitcheck(bitstr, n)]
+
+ assert expand(ei2.bitstring_readonly_descrs_fields) == [f1descr.ei_index]
+ assert expand(ei2.bitstring_write_descrs_fields) == []
+ assert expand(ei2.bitstring_readonly_descrs_arrays) == []
+ assert expand(ei2.bitstring_write_descrs_arrays) == []
+
+ assert expand(ei3.bitstring_readonly_descrs_fields) == [f1descr.ei_index]
+ assert expand(ei3.bitstring_write_descrs_fields) == [f2descr.ei_index]
+ assert expand(ei3.bitstring_readonly_descrs_arrays) == [0] #a1descr,a2descr
+ assert expand(ei3.bitstring_write_descrs_arrays) == []
+
+ for ei in [ei2, ei3]:
+ for fdescr in [f1descr, f2descr]:
+ assert ei.check_readonly_descr_field(fdescr) == (
+ fdescr in ei._readonly_descrs_fields)
+ assert ei.check_write_descr_field(fdescr) == (
+ fdescr in ei._write_descrs_fields)
+ for adescr in [a1descr, a2descr]:
+ assert ei.check_readonly_descr_array(adescr) == (
+ adescr in ei._readonly_descrs_arrays)
+ assert ei.check_write_descr_array(adescr) == (
+ adescr in ei._write_descrs_arrays)
diff --git a/rpython/jit/metainterp/heapcache.py
b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -209,7 +209,7 @@
isinstance(argboxes[3], ConstInt) and
isinstance(argboxes[4], ConstInt) and
isinstance(argboxes[5], ConstInt) and
- len(descr.get_extra_info().write_descrs_arrays) == 1):
+ descr.get_extra_info().single_write_descr_array is not None):
# ARRAYCOPY with constant starts and constant length doesn't escape
# its argument
# XXX really?
@@ -299,9 +299,9 @@
isinstance(argboxes[3], ConstInt) and
isinstance(argboxes[4], ConstInt) and
isinstance(argboxes[5], ConstInt) and
- len(effectinfo.write_descrs_arrays) == 1
+ effectinfo.single_write_descr_array is not None
):
- descr = effectinfo.write_descrs_arrays[0]
+ descr = effectinfo.single_write_descr_array
cache = self.heap_array_cache.get(descr, None)
srcstart = argboxes[3].getint()
dststart = argboxes[4].getint()
@@ -328,10 +328,10 @@
idx_cache._clear_cache_on_write(seen_allocation_of_target)
return
elif (
- len(effectinfo.write_descrs_arrays) == 1
+ effectinfo.single_write_descr_array is not None
):
# Fish the descr out of the effectinfo
- cache =
self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
+ cache =
self.heap_array_cache.get(effectinfo.single_write_descr_array, None)
if cache is not None:
for idx, cache in cache.iteritems():
cache._clear_cache_on_write(seen_allocation_of_target)
diff --git a/rpython/jit/metainterp/history.py
b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -1,3 +1,4 @@
+import sys
from rpython.rtyper.extregistry import ExtRegistryEntry
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from rpython.rlib.objectmodel import we_are_translated, Symbolic
@@ -87,9 +88,10 @@
class AbstractDescr(AbstractValue):
- __slots__ = ('descr_index',)
+ __slots__ = ('descr_index', 'ei_index')
llopaque = True
descr_index = -1
+ ei_index = sys.maxint
def repr_of_descr(self):
return '%r' % (self,)
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py
b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -432,28 +432,35 @@
optimize_GUARD_EXCEPTION = optimize_GUARD_NO_EXCEPTION
def force_from_effectinfo(self, effectinfo):
- # XXX we can get the wrong complexity here, if the lists
- # XXX stored on effectinfo are large
- for fielddescr in effectinfo.readonly_descrs_fields:
- self.force_lazy_set(fielddescr)
- for arraydescr in effectinfo.readonly_descrs_arrays:
- self.force_lazy_setarrayitem(arraydescr)
- for fielddescr in effectinfo.write_descrs_fields:
- if fielddescr.is_always_pure():
- continue
- try:
- del self.cached_dict_reads[fielddescr]
- except KeyError:
- pass
- self.force_lazy_set(fielddescr, can_cache=False)
- for arraydescr in effectinfo.write_descrs_arrays:
- self.force_lazy_setarrayitem(arraydescr, can_cache=False)
- if arraydescr in self.corresponding_array_descrs:
- dictdescr = self.corresponding_array_descrs.pop(arraydescr)
+ # Note: this version of the code handles effectively
+ # effectinfos that store arbitrarily many descrs, by looping
+ # on self.cached_{fields, arrayitems} and looking them up in
+ # the bitstrings stored in the effectinfo.
+ for fielddescr, cf in self.cached_fields.items():
+ if effectinfo.check_readonly_descr_field(fielddescr):
+ cf.force_lazy_set(self, fielddescr)
+ if effectinfo.check_write_descr_field(fielddescr):
+ if fielddescr.is_always_pure():
+ continue
+ try:
+ del self.cached_dict_reads[fielddescr]
+ except KeyError:
+ pass
+ cf.force_lazy_set(self, fielddescr, can_cache=False)
+ #
+ for arraydescr, submap in self.cached_arrayitems.items():
+ if effectinfo.check_readonly_descr_array(arraydescr):
+ self.force_lazy_setarrayitem_submap(submap)
+ if effectinfo.check_write_descr_array(arraydescr):
+ self.force_lazy_setarrayitem_submap(submap, can_cache=False)
+ #
+ for arraydescr, dictdescr in self.corresponding_array_descrs.items():
+ if effectinfo.check_write_descr_array(arraydescr):
try:
del self.cached_dict_reads[dictdescr]
except KeyError:
pass # someone did it already
+ #
if effectinfo.check_forces_virtual_or_virtualizable():
vrefinfo = self.optimizer.metainterp_sd.virtualref_info
self.force_lazy_set(vrefinfo.descr_forced)
@@ -476,6 +483,10 @@
if indexb is None or indexb.contains(idx):
cf.force_lazy_set(self, None, can_cache)
+ def force_lazy_setarrayitem_submap(self, submap, can_cache=True):
+ for cf in submap.itervalues():
+ cf.force_lazy_set(self, None, can_cache)
+
def force_all_lazy_sets(self):
items = self.cached_fields.items()
if not we_are_translated():
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py
b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -620,10 +620,10 @@
and length and ((dest_info and dest_info.is_virtual()) or
length.getint() <= 8) and
((source_info and source_info.is_virtual()) or length.getint() <=
8)
- and len(extrainfo.write_descrs_arrays) == 1): # <-sanity check
+ and extrainfo.single_write_descr_array is not None): #<-sanity
check
source_start = source_start_box.getint()
dest_start = dest_start_box.getint()
- arraydescr = extrainfo.write_descrs_arrays[0]
+ arraydescr = extrainfo.single_write_descr_array
if arraydescr.is_array_of_structs():
return False # not supported right now
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py
b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -10,7 +10,7 @@
from rpython.jit.metainterp.history import (TreeLoop, AbstractDescr,
JitCellToken, TargetToken)
from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
-from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter.effectinfo import EffectInfo, compute_bitstrings
from rpython.jit.metainterp.logger import LogOperations
from rpython.jit.tool.oparser import OpParser, pure_parse,
convert_loop_to_trace
from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
@@ -530,6 +530,7 @@
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
+ compute_bitstrings(self.cpu.fetch_all_descrs())
#
compile_data.enable_opts = self.enable_opts
state = optimize_trace(metainterp_sd, None, compile_data)
diff --git a/rpython/jit/metainterp/pyjitpl.py
b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1838,7 +1838,11 @@
self.cpu.propagate_exception_descr = exc_descr
#
self.globaldata = MetaInterpGlobalData(self)
+
+ def finish_setup_descrs(self):
+ from rpython.jit.codewriter import effectinfo
self.all_descrs = self.cpu.setup_descrs()
+ effectinfo.compute_bitstrings(self.all_descrs)
def _setup_once(self):
"""Runtime setup needed by the various components of the JIT."""
diff --git a/rpython/jit/metainterp/test/support.py
b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -132,6 +132,7 @@
metainterp_sd = pyjitpl.MetaInterpStaticData(cw.cpu, opt)
stats.metainterp_sd = metainterp_sd
metainterp_sd.finish_setup(cw)
+ metainterp_sd.finish_setup_descrs()
[jitdriver_sd] = metainterp_sd.jitdrivers_sd
metainterp = pyjitpl.MetaInterp(metainterp_sd, jitdriver_sd)
diff --git a/rpython/jit/metainterp/test/test_heapcache.py
b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -27,8 +27,12 @@
def __init__(self, extraeffect, oopspecindex, write_descrs_fields,
write_descrs_arrays):
self.extraeffect = extraeffect
self.oopspecindex = oopspecindex
- self.write_descrs_fields = write_descrs_fields
- self.write_descrs_arrays = write_descrs_arrays
+ self._write_descrs_fields = write_descrs_fields
+ self._write_descrs_arrays = write_descrs_arrays
+ if len(write_descrs_arrays) == 1:
+ [self.single_write_descr_array] = write_descrs_arrays
+ else:
+ self.single_write_descr_array = None
def has_random_effects(self):
return self.extraeffect == self.EF_RANDOM_EFFECTS
@@ -37,14 +41,14 @@
def __init__(self, extraeffect, oopspecindex=None, write_descrs_fields=[],
write_descrs_arrays=[]):
self.extraeffect = extraeffect
self.oopspecindex = oopspecindex
- self.write_descrs_fields = write_descrs_fields
- self.write_descrs_arrays = write_descrs_arrays
+ self.__write_descrs_fields = write_descrs_fields
+ self.__write_descrs_arrays = write_descrs_arrays
def get_extra_info(self):
return FakeEffectinfo(
self.extraeffect, self.oopspecindex,
- write_descrs_fields=self.write_descrs_fields,
- write_descrs_arrays=self.write_descrs_arrays,
+ write_descrs_fields=self.__write_descrs_fields,
+ write_descrs_arrays=self.__write_descrs_arrays,
)
arraycopydescr1 = FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE,
FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1])
diff --git a/rpython/jit/metainterp/test/test_warmspot.py
b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -624,7 +624,7 @@
pass
def setup_descrs(self):
- pass
+ return []
def get_latest_descr(self, deadframe):
assert isinstance(deadframe, FakeDeadFrame)
diff --git a/rpython/jit/metainterp/warmspot.py
b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -277,6 +277,7 @@
for vinfo in vinfos:
if vinfo is not None:
vinfo.finish()
+ self.metainterp_sd.finish_setup_descrs()
if self.cpu.translate_support_code:
self.annhelper.finish()
diff --git a/rpython/tool/algo/bitstring.py b/rpython/tool/algo/bitstring.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/algo/bitstring.py
@@ -0,0 +1,23 @@
+
+
+def make_bitstring(lst):
+ "NOT_RPYTHON"
+ if not lst:
+ return ''
+ num_bits = max(lst) + 1
+ num_bytes = (num_bits + 7) // 8
+ entries = [0] * num_bytes
+ for x in lst:
+ assert x >= 0
+ entries[x >> 3] |= 1 << (x & 7)
+ return ''.join(map(chr, entries))
+
+def bitcheck(bitstring, n):
+ assert n >= 0
+ byte_number = n >> 3
+ if byte_number >= len(bitstring):
+ return False
+ return (ord(bitstring[byte_number]) & (1 << (n & 7))) != 0
+
+def num_bits(bitstring):
+ return len(bitstring) << 3
diff --git a/rpython/tool/algo/test/test_bitstring.py
b/rpython/tool/algo/test/test_bitstring.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/algo/test/test_bitstring.py
@@ -0,0 +1,25 @@
+from rpython.tool.algo.bitstring import *
+from hypothesis import given, strategies
+
+def test_make():
+ assert make_bitstring([]) == ''
+ assert make_bitstring([0]) == '\x01'
+ assert make_bitstring([7]) == '\x80'
+ assert make_bitstring([8]) == '\x00\x01'
+ assert make_bitstring([2, 4, 20]) == '\x14\x00\x10'
+
+def test_bitcheck():
+ assert bitcheck('\x01', 0) is True
+ assert bitcheck('\x01', 1) is False
+ assert bitcheck('\x01', 10) is False
+ assert [n for n in range(32) if bitcheck('\x14\x00\x10', n)] == [2, 4, 20]
+
+@given(strategies.lists(strategies.integers(min_value=0, max_value=299)))
+def test_random(lst):
+ bitstring = make_bitstring(lst)
+ assert set([n for n in range(300) if bitcheck(bitstring, n)]) == set(lst)
+
+def test_num_bits():
+ assert num_bits('') == 0
+ assert num_bits('a') == 8
+ assert num_bits('bcd') == 24
diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py
b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -1,3 +1,4 @@
+import py
from rpython.rtyper.lltypesystem import lltype
from rpython.translator.translator import TranslationContext, graphof
from rpython.translator.backendopt.writeanalyze import WriteAnalyzer, top_set
@@ -314,6 +315,7 @@
assert T1 == T2
def test_cutoff(self):
+ py.test.skip("cutoff: disabled")
from rpython.rlib.unroll import unrolling_iterable
cutoff = 20
attrs = unrolling_iterable(["s%s" % i for i in range(cutoff + 5)])
diff --git a/rpython/translator/backendopt/writeanalyze.py
b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -4,10 +4,14 @@
top_set = object()
empty_set = frozenset()
-CUTOFF = 3000
+# CUTOFF is disabled, as it gave a strangely not-working-any-more effect
+# if the size of the result grows past that bound. The main user was
+# optimizeopt/heap.py (force_from_effectinfo), which has been rewritten
+# to be happy with any size now.
+#CUTOFF = 3000
class WriteAnalyzer(graphanalyze.GraphAnalyzer):
- cutoff = CUTOFF
+ #cutoff = CUTOFF
def bottom_result(self):
return empty_set
@@ -25,8 +29,8 @@
if other is top_set:
return top_set
result.update(other)
- if len(result) > self.cutoff:
- return top_set
+ #if len(result) > self.cutoff:
+ # return top_set
return result
def finalize_builder(self, result):
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
