Author: Matti Picus <matti.pi...@gmail.com>
Branch: py3.5
Changeset: r95446:1300ca1763e1
Date: 2018-12-09 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/1300ca1763e1/

Log:    merge default into branch

diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py 
b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py
--- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py
+++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py
@@ -4,7 +4,12 @@
 from cffi import cffi_opcode
 
 if '__pypy__' in sys.builtin_module_names:
-    py.test.skip("not available on pypy", allow_module_level=True)
+    try:
+        # pytest >= 4.0
+        py.test.skip("not available on pypy", allow_module_level=True)
+    except TypeError:
+        # older pytest
+        py.test.skip("not available on pypy")
 
 cffi_dir = os.path.dirname(cffi_opcode.__file__)
 
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -4,8 +4,10 @@
 from errno import EINVAL, EPERM
 import _structseq, os
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 
 class error(Exception):
@@ -35,7 +37,7 @@
     ru_oublock = _structseq.structseqfield(10, "block output operations")
     ru_msgsnd = _structseq.structseqfield(11,  "IPC messages sent")
     ru_msgrcv = _structseq.structseqfield(12,  "IPC messages received")
-    ru_nsignals = _structseq.structseqfield(13,"signals received")
+    ru_nsignals = _structseq.structseqfield(13, "signals received")
     ru_nvcsw = _structseq.structseqfield(14,   "voluntary context switches")
     ru_nivcsw = _structseq.structseqfield(15,  "involuntary context switches")
 
@@ -57,7 +59,7 @@
         ru.ru_nsignals,
         ru.ru_nvcsw,
         ru.ru_nivcsw,
-        ))
+    ))
 
 @builtinify
 def getrusage(who):
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -51,3 +51,8 @@
 .. branch: rlock-in-rpython
 
 Backport CPython fix for `thread.RLock` 
+
+
+.. branch: expose-gc-time
+
+Make GC hooks measure time in seconds (as opposed to an opaque unit).
diff --git a/pypy/goal/targetpypystandalone.py 
b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -386,7 +386,7 @@
     def get_gchooks(self):
         from pypy.module.gc.hook import LowLevelGcHooks
         if self.space is None:
-            raise Exception("get_gchooks must be called afeter 
get_entry_point")
+            raise Exception("get_gchooks must be called after get_entry_point")
         return self.space.fromcache(LowLevelGcHooks)
 
     def get_entry_point(self, config):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -603,8 +603,14 @@
         warnoptions = pythonwarnings.split(',') + warnoptions
     if warnoptions:
         sys.warnoptions[:] = warnoptions
-        from warnings import _processoptions
-        _processoptions(sys.warnoptions)
+        try:
+            if 'warnings' in sys.modules:
+                from warnings import _processoptions
+                _processoptions(sys.warnoptions)
+            else:
+                import warnings
+        except ImportError as e:
+            pass   # CPython just eats any exception here
 
     # set up the Ctrl-C => KeyboardInterrupt signal handler, if the
     # signal module is available
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -57,12 +57,14 @@
                      'total_allocated_memory', 'jit_backend_allocated',
                      'peak_memory', 'peak_allocated_memory', 
'total_arena_memory',
                      'total_rawmalloced_memory', 'nursery_size',
-                     'peak_arena_memory', 'peak_rawmalloced_memory'):
+                     'peak_arena_memory', 'peak_rawmalloced_memory',
+                     ):
             setattr(self, item, self._format(getattr(self._s, item)))
         self.memory_used_sum = self._format(self._s.total_gc_memory + 
self._s.total_memory_pressure +
                                             self._s.jit_backend_used)
         self.memory_allocated_sum = 
self._format(self._s.total_allocated_memory + self._s.total_memory_pressure +
                                             self._s.jit_backend_allocated)
+        self.total_gc_time = self._s.total_gc_time
 
     def _format(self, v):
         if v < 1000000:
@@ -92,6 +94,8 @@
     raw assembler allocated: %s%s
     -----------------------------
     Total:                   %s
+
+    Total time spent in GC:  %s
     """ % (self.total_gc_memory, self.peak_memory,
               self.total_arena_memory,
               self.total_rawmalloced_memory,
@@ -106,7 +110,8 @@
               self.nursery_size,
            self.jit_backend_allocated,
            extra,
-           self.memory_allocated_sum)
+           self.memory_allocated_sum,
+           self.total_gc_time / 1000.0)
 
 
 def get_stats(memory_pressure=False):
diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -7,6 +7,8 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, 
GetSetProperty
 from pypy.interpreter.executioncontext import AsyncAction
 
+inf = float("inf")
+
 class LowLevelGcHooks(GcHooks):
     """
     These are the low-level hooks which are called directly from the GC.
@@ -126,9 +128,9 @@
 
     def reset(self):
         self.count = 0
-        self.duration = r_longlong(0)
-        self.duration_min = r_longlong(longlongmax)
-        self.duration_max = r_longlong(0)
+        self.duration = 0.0
+        self.duration_min = inf
+        self.duration_max = 0.0
 
     def fix_annotation(self):
         # the annotation of the class and its attributes must be completed
@@ -136,9 +138,9 @@
         # annotated with the correct types
         if NonConstant(False):
             self.count = NonConstant(-42)
-            self.duration = NonConstant(r_longlong(-42))
-            self.duration_min = NonConstant(r_longlong(-42))
-            self.duration_max = NonConstant(r_longlong(-42))
+            self.duration = NonConstant(-53.2)
+            self.duration_min = NonConstant(-53.2)
+            self.duration_max = NonConstant(-53.2)
             self.total_memory_used = NonConstant(r_uint(42))
             self.pinned_objects = NonConstant(-42)
             self.fire()
@@ -166,9 +168,9 @@
 
     def reset(self):
         self.count = 0
-        self.duration = r_longlong(0)
-        self.duration_min = r_longlong(longlongmax)
-        self.duration_max = r_longlong(0)
+        self.duration = 0.0
+        self.duration_min = inf
+        self.duration_max = 0.0
 
     def fix_annotation(self):
         # the annotation of the class and its attributes must be completed
@@ -176,9 +178,9 @@
         # annotated with the correct types
         if NonConstant(False):
             self.count = NonConstant(-42)
-            self.duration = NonConstant(r_longlong(-42))
-            self.duration_min = NonConstant(r_longlong(-42))
-            self.duration_max = NonConstant(r_longlong(-42))
+            self.duration = NonConstant(-53.2)
+            self.duration_min = NonConstant(-53.2)
+            self.duration_max = NonConstant(-53.2)
             self.oldstate = NonConstant(-42)
             self.newstate = NonConstant(-42)
             self.fire()
@@ -276,10 +278,14 @@
 
 
 # just a shortcut to make the typedefs shorter
-def wrap_many_ints(cls, names):
+def wrap_many(cls, names):
     d = {}
     for name in names:
-        d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint")
+        if "duration" in name:
+            wrapfn = "newfloat"
+        else:
+            wrapfn = "newint"
+        d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn)
     return d
 
 
@@ -303,7 +309,7 @@
 
 W_GcMinorStats.typedef = TypeDef(
     "GcMinorStats",
-    **wrap_many_ints(W_GcMinorStats, (
+    **wrap_many(W_GcMinorStats, (
         "count",
         "duration",
         "duration_min",
@@ -319,7 +325,7 @@
     STATE_SWEEPING = incminimark.STATE_SWEEPING,
     STATE_FINALIZING = incminimark.STATE_FINALIZING,
     GC_STATES = tuple(incminimark.GC_STATES),
-    **wrap_many_ints(W_GcCollectStepStats, (
+    **wrap_many(W_GcCollectStepStats, (
         "count",
         "duration",
         "duration_min",
@@ -330,7 +336,7 @@
 
 W_GcCollectStats.typedef = TypeDef(
     "GcCollectStats",
-    **wrap_many_ints(W_GcCollectStats, (
+    **wrap_many(W_GcCollectStats, (
         "count",
         "num_major_collects",
         "arenas_count_before",
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -189,6 +189,7 @@
         self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY)
         self.peak_rawmalloced_memory = 
rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY)
         self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE)
+        self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME)
 
 W_GcStats.typedef = TypeDef("GcStats",
     total_memory_pressure=interp_attrproperty("total_memory_pressure",
@@ -215,6 +216,8 @@
         cls=W_GcStats, wrapfn="newint"),
     nursery_size=interp_attrproperty("nursery_size",
         cls=W_GcStats, wrapfn="newint"),
+    total_gc_time=interp_attrproperty("total_gc_time",
+        cls=W_GcStats, wrapfn="newint"),
 )
 
 @unwrap_spec(memory_pressure=bool)
diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py
--- a/pypy/module/gc/test/test_hook.py
+++ b/pypy/module/gc/test/test_hook.py
@@ -26,11 +26,11 @@
 
         @unwrap_spec(ObjSpace)
         def fire_many(space):
-            gchooks.fire_gc_minor(5, 0, 0)
-            gchooks.fire_gc_minor(7, 0, 0)
-            gchooks.fire_gc_collect_step(5, 0, 0)
-            gchooks.fire_gc_collect_step(15, 0, 0)
-            gchooks.fire_gc_collect_step(22, 0, 0)
+            gchooks.fire_gc_minor(5.0, 0, 0)
+            gchooks.fire_gc_minor(7.0, 0, 0)
+            gchooks.fire_gc_collect_step(5.0, 0, 0)
+            gchooks.fire_gc_collect_step(15.0, 0, 0)
+            gchooks.fire_gc_collect_step(22.0, 0, 0)
             gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6)
 
         cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor))
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py 
b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -5,327 +5,321 @@
 import pytest
 import sys
 
+_sqlite3 = pytest.importorskip('_sqlite3')
 
-def pytest_funcarg__con(request):
+pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
+    reason="PyPy-only test")
+
+
+@pytest.yield_fixture
+def con():
     con = _sqlite3.connect(':memory:')
-    request.addfinalizer(lambda: con.close())
-    return con
+    yield con
+    con.close()
 
 
-class BaseTestSQLite:
-    def test_list_ddl(self, con):
-        """From issue996.  Mostly just looking for lack of exceptions."""
-        cursor = con.cursor()
-        cursor.execute('CREATE TABLE foo (bar INTEGER)')
-        result = list(cursor)
-        assert result == []
-        cursor.execute('INSERT INTO foo (bar) VALUES (42)')
-        result = list(cursor)
-        assert result == []
-        cursor.execute('SELECT * FROM foo')
-        result = list(cursor)
-        assert result == [(42,)]
+def test_list_ddl(con):
+    """From issue996.  Mostly just looking for lack of exceptions."""
+    cursor = con.cursor()
+    cursor.execute('CREATE TABLE foo (bar INTEGER)')
+    result = list(cursor)
+    assert result == []
+    cursor.execute('INSERT INTO foo (bar) VALUES (42)')
+    result = list(cursor)
+    assert result == []
+    cursor.execute('SELECT * FROM foo')
+    result = list(cursor)
+    assert result == [(42,)]
 
-    def test_connect_takes_same_positional_args_as_Connection(self, con):
-        if not hasattr(_sqlite3, '_ffi'):
-            pytest.skip("only works for lib_pypy _sqlite3")
-        from inspect import getargspec
-        clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore 
self
-        conargs = getargspec(_sqlite3.connect).args
-        assert clsargs == conargs
+@pypy_only
+def test_connect_takes_same_positional_args_as_Connection(con):
+    from inspect import getargspec
+    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+    conargs = getargspec(_sqlite3.connect).args
+    assert clsargs == conargs
 
-    def test_total_changes_after_close(self, con):
-        con.close()
-        pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+def test_total_changes_after_close(con):
+    con.close()
+    with pytest.raises(_sqlite3.ProgrammingError):
+        con.total_changes
 
-    def test_connection_check_init(self):
-        class Connection(_sqlite3.Connection):
-            def __init__(self, name):
-                pass
+def test_connection_check_init():
+    class Connection(_sqlite3.Connection):
+        def __init__(self, name):
+            pass
 
-        con = Connection(":memory:")
-        e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
-        assert '__init__' in str(e.value)
+    con = Connection(":memory:")
+    with pytest.raises(_sqlite3.ProgrammingError) as excinfo:
+        con.cursor()
+    assert '__init__' in str(excinfo.value)
 
-    def test_cursor_check_init(self, con):
-        class Cursor(_sqlite3.Cursor):
-            def __init__(self, name):
-                pass
 
-        cur = Cursor(con)
-        e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
-        assert '__init__' in str(e.value)
+def test_cursor_check_init(con):
+    class Cursor(_sqlite3.Cursor):
+        def __init__(self, name):
+            pass
 
-    def test_connection_after_close(self, con):
-        pytest.raises(TypeError, "con()")
-        con.close()
-        # raises ProgrammingError because should check closed before check args
-        pytest.raises(_sqlite3.ProgrammingError, "con()")
+    cur = Cursor(con)
+    with pytest.raises(_sqlite3.ProgrammingError) as excinfo:
+        cur.execute('select 1')
+    assert '__init__' in str(excinfo.value)
 
-    def test_cursor_iter(self, con):
+def test_connection_after_close(con):
+    with pytest.raises(TypeError):
+        con()
+    con.close()
+    # raises ProgrammingError because should check closed before check args
+    with pytest.raises(_sqlite3.ProgrammingError):
+        con()
+
+def test_cursor_iter(con):
+    cur = con.cursor()
+    with pytest.raises(StopIteration):
+        next(cur)
+
+    cur.execute('select 1')
+    next(cur)
+    with pytest.raises(StopIteration):
+        next(cur)
+
+    cur.execute('select 1')
+    con.commit()
+    next(cur)
+    with pytest.raises(StopIteration):
+        next(cur)
+
+    with pytest.raises(_sqlite3.ProgrammingError):
+        cur.executemany('select 1', [])
+    with pytest.raises(StopIteration):
+        next(cur)
+
+    cur.execute('select 1')
+    cur.execute('create table test(ing)')
+    with pytest.raises(StopIteration):
+        next(cur)
+
+    cur.execute('select 1')
+    cur.execute('insert into test values(1)')
+    con.commit()
+    with pytest.raises(StopIteration):
+        next(cur)
+
+def test_cursor_after_close(con):
+    cur = con.execute('select 1')
+    cur.close()
+    con.close()
+    with pytest.raises(_sqlite3.ProgrammingError):
+        cur.close()
+    # raises ProgrammingError because should check closed before check args
+    with pytest.raises(_sqlite3.ProgrammingError):
+        cur.execute(1,2,3,4,5)
+    with pytest.raises(_sqlite3.ProgrammingError):
+        cur.executemany(1,2,3,4,5)
+
+@pypy_only
+def test_connection_del(tmpdir):
+    """For issue1325."""
+    import os
+    import gc
+    resource = pytest.importorskip('resource')
+
+    limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+    try:
+        fds = 0
+        while True:
+            fds += 1
+            resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
+            try:
+                for p in os.pipe(): os.close(p)
+            except OSError:
+                assert fds < 100
+            else:
+                break
+
+        def open_many(cleanup):
+            con = []
+            for i in range(3):
+                con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
+                if cleanup:
+                    con[i] = None
+                    gc.collect(); gc.collect()
+
+        with pytest.raises(_sqlite3.OperationalError):
+            open_many(False)
+        sys.exc_clear()
+        gc.collect(); gc.collect()
+        open_many(True)
+    finally:
+        resource.setrlimit(resource.RLIMIT_NOFILE, limit)
+
+def test_on_conflict_rollback_executemany(con):
+    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+    if (int(major), int(minor), int(micro)) < (3, 2, 2):
+        pytest.skip("requires sqlite3 version >= 3.2.2")
+    con.execute("create table foo(x, unique(x) on conflict rollback)")
+    con.execute("insert into foo(x) values (1)")
+    try:
+        con.executemany("insert into foo(x) values (?)", [[1]])
+    except _sqlite3.DatabaseError:
+        pass
+    con.execute("insert into foo(x) values (2)")
+    try:
+        con.commit()
+    except _sqlite3.OperationalError:
+        pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
+
+def test_statement_arg_checking(con):
+    with pytest.raises(_sqlite3.Warning) as e:
+        con(123)
+    assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
+    with pytest.raises(ValueError) as e:
+        con.execute(123)
+    assert str(e.value) == 'operation parameter must be str or unicode'
+    with pytest.raises(ValueError) as e:
+        con.executemany(123, 123)
+    assert str(e.value) == 'operation parameter must be str or unicode'
+    with pytest.raises(ValueError) as e:
+        con.executescript(123)
+    assert str(e.value) == 'script argument must be unicode or string.'
+
+def test_statement_param_checking(con):
+    con.execute('create table foo(x)')
+    con.execute('insert into foo(x) values (?)', [2])
+    con.execute('insert into foo(x) values (?)', (2,))
+    class seq(object):
+        def __len__(self):
+            return 1
+        def __getitem__(self, key):
+            return 2
+    con.execute('insert into foo(x) values (?)', seq())
+    del seq.__len__
+    with pytest.raises(_sqlite3.ProgrammingError):
+        con.execute('insert into foo(x) values (?)', seq())
+    with pytest.raises(_sqlite3.ProgrammingError):
+        con.execute('insert into foo(x) values (?)', {2:2})
+    with pytest.raises(ValueError) as e:
+        con.execute('insert into foo(x) values (?)', 2)
+    assert str(e.value) == 'parameters are of unsupported type'
+
+def test_explicit_begin(con):
+    con.execute('BEGIN')
+    con.execute('BEGIN ')
+    con.execute('BEGIN')
+    con.commit()
+    con.execute('BEGIN')
+    con.commit()
+
+def test_row_factory_use(con):
+    con.row_factory = 42
+    con.execute('select 1')
+
+def test_returning_blob_must_own_memory(con):
+    import gc
+    con.create_function("returnblob", 0, lambda: buffer("blob"))
+    cur = con.execute("select returnblob()")
+    val = cur.fetchone()[0]
+    for i in range(5):
+        gc.collect()
+        got = (val[0], val[1], val[2], val[3])
+        assert got == ('b', 'l', 'o', 'b')
+    # in theory 'val' should be a read-write buffer
+    # but it's not right now
+    if not hasattr(_sqlite3, '_ffi'):
+        val[1] = 'X'
+        got = (val[0], val[1], val[2], val[3])
+        assert got == ('b', 'X', 'o', 'b')
+
+def test_description_after_fetchall(con):
+    cur = con.cursor()
+    assert cur.description is None
+    cur.execute("select 42").fetchall()
+    assert cur.description is not None
+
+def test_executemany_lastrowid(con):
+    cur = con.cursor()
+    cur.execute("create table test(a)")
+    cur.executemany("insert into test values (?)", [[1], [2], [3]])
+    assert cur.lastrowid is None
+    # issue 2682
+    cur.execute('''insert
+                into test
+                values (?)
+                ''', (1, ))
+    assert cur.lastrowid is not None
+    cur.execute('''insert\t into test values (?) ''', (1, ))
+    assert cur.lastrowid is not None
+
+def test_authorizer_bad_value(con):
+    def authorizer_cb(action, arg1, arg2, dbname, source):
+        return 42
+    con.set_authorizer(authorizer_cb)
+    with pytest.raises(_sqlite3.OperationalError) as e:
+        con.execute('select 123')
+    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+    if (int(major), int(minor), int(micro)) >= (3, 6, 14):
+        assert str(e.value) == 'authorizer malfunction'
+    else:
+        assert str(e.value) == \
+            ("illegal return value (1) from the authorization function - "
+                "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
+
+def test_issue1573(con):
+    cur = con.cursor()
+    cur.execute(u'SELECT 1 as m&#233;il')
+    assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
+
+def test_adapter_exception(con):
+    def cast(obj):
+        raise ZeroDivisionError
+
+    _sqlite3.register_adapter(int, cast)
+    try:
         cur = con.cursor()
-        with pytest.raises(StopIteration):
-            next(cur)
+        cur.execute("select ?", (4,))
+        val = cur.fetchone()[0]
+        # Adapter error is ignored, and parameter is passed as is.
+        assert val == 4
+        assert type(val) is int
+    finally:
+        del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
 
-        cur.execute('select 1')
-        next(cur)
-        with pytest.raises(StopIteration):
-            next(cur)
+def test_null_character(con):
+    if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9):
+        pytest.skip("_sqlite3 too old")
+    with raises(ValueError) as excinfo:
+        con("\0select 1")
+    assert str(excinfo.value) == "the query contains a null character"
+    with raises(ValueError) as excinfo:
+        con("select 1\0")
+    assert str(excinfo.value) == "the query contains a null character"
+    cur = con.cursor()
+    with raises(ValueError) as excinfo:
+        cur.execute("\0select 2")
+    assert str(excinfo.value) == "the query contains a null character"
+    with raises(ValueError) as excinfo:
+        cur.execute("select 2\0")
+    assert str(excinfo.value) == "the query contains a null character"
 
-        cur.execute('select 1')
-        con.commit()
-        next(cur)
-        with pytest.raises(StopIteration):
-            next(cur)
-
-        with pytest.raises(_sqlite3.ProgrammingError):
-            cur.executemany('select 1', [])
-        with pytest.raises(StopIteration):
-            next(cur)
-
-        cur.execute('select 1')
-        cur.execute('create table test(ing)')
-        with pytest.raises(StopIteration):
-            next(cur)
-
-        cur.execute('select 1')
-        cur.execute('insert into test values(1)')
-        con.commit()
-        with pytest.raises(StopIteration):
-            next(cur)
-
-    def test_cursor_after_close(self, con):
-        cur = con.execute('select 1')
-        cur.close()
-        con.close()
-        pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
-        # raises ProgrammingError because should check closed before check args
-        pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
-        pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
-
-    @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
-    def test_connection_del(self, tmpdir):
-        """For issue1325."""
-        import os
-        import gc
-        try:
-            import resource
-        except ImportError:
-            pytest.skip("needs resource module")
-
-        limit = resource.getrlimit(resource.RLIMIT_NOFILE)
-        try:
-            fds = 0
-            while True:
-                fds += 1
-                resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
-                try:
-                    for p in os.pipe(): os.close(p)
-                except OSError:
-                    assert fds < 100
-                else:
-                    break
-
-            def open_many(cleanup):
-                con = []
-                for i in range(3):
-                    con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
-                    if cleanup:
-                        con[i] = None
-                        gc.collect(); gc.collect()
-
-            pytest.raises(_sqlite3.OperationalError, open_many, False)
-            gc.collect(); gc.collect()
-            open_many(True)
-        finally:
-            resource.setrlimit(resource.RLIMIT_NOFILE, limit)
-
-    def test_on_conflict_rollback_executemany(self, con):
-        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-        if (int(major), int(minor), int(micro)) < (3, 2, 2):
-            pytest.skip("requires sqlite3 version >= 3.2.2")
-        con.execute("create table foo(x, unique(x) on conflict rollback)")
-        con.execute("insert into foo(x) values (1)")
-        try:
-            con.executemany("insert into foo(x) values (?)", [[1]])
-        except _sqlite3.DatabaseError:
-            pass
-        con.execute("insert into foo(x) values (2)")
-        try:
-            con.commit()
-        except _sqlite3.OperationalError:
-            pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
-
-    def test_statement_arg_checking(self, con):
-        with pytest.raises(_sqlite3.Warning) as e:
-            con(123)
-        assert str(e.value) == 'SQL is of wrong type. Must be string or 
unicode.'
-        with pytest.raises(ValueError) as e:
-            con.execute(123)
-        assert str(e.value) == 'operation parameter must be str or unicode'
-        with pytest.raises(ValueError) as e:
-            con.executemany(123, 123)
-        assert str(e.value) == 'operation parameter must be str or unicode'
-        with pytest.raises(ValueError) as e:
-            con.executescript(123)
-        assert str(e.value) == 'script argument must be unicode or string.'
-
-    def test_statement_param_checking(self, con):
-        con.execute('create table foo(x)')
-        con.execute('insert into foo(x) values (?)', [2])
-        con.execute('insert into foo(x) values (?)', (2,))
-        class seq(object):
-            def __len__(self):
-                return 1
-            def __getitem__(self, key):
-                return 2
-        con.execute('insert into foo(x) values (?)', seq())
-        del seq.__len__
-        with pytest.raises(_sqlite3.ProgrammingError):
-            con.execute('insert into foo(x) values (?)', seq())
-        with pytest.raises(_sqlite3.ProgrammingError):
-            con.execute('insert into foo(x) values (?)', {2:2})
-        with pytest.raises(ValueError) as e:
-            con.execute('insert into foo(x) values (?)', 2)
-        assert str(e.value) == 'parameters are of unsupported type'
-
-    def test_explicit_begin(self, con):
-        con.execute('BEGIN')
-        con.execute('BEGIN ')
-        con.execute('BEGIN')
-        con.commit()
-        con.execute('BEGIN')
-        con.commit()
-
-    def test_row_factory_use(self, con):
-        con.row_factory = 42
-        con.execute('select 1')
-
-    def test_returning_blob_must_own_memory(self, con):
-        import gc
-        con.create_function("returnblob", 0, lambda: buffer("blob"))
-        cur = con.execute("select returnblob()")
-        val = cur.fetchone()[0]
-        for i in range(5):
-            gc.collect()
-            got = (val[0], val[1], val[2], val[3])
-            assert got == ('b', 'l', 'o', 'b')
-        # in theory 'val' should be a read-write buffer
-        # but it's not right now
-        if not hasattr(_sqlite3, '_ffi'):
-            val[1] = 'X'
-            got = (val[0], val[1], val[2], val[3])
-            assert got == ('b', 'X', 'o', 'b')
-
-    def test_description_after_fetchall(self, con):
-        cur = con.cursor()
-        assert cur.description is None
-        cur.execute("select 42").fetchall()
-        assert cur.description is not None
-
-    def test_executemany_lastrowid(self, con):
-        cur = con.cursor()
-        cur.execute("create table test(a)")
-        cur.executemany("insert into test values (?)", [[1], [2], [3]])
-        assert cur.lastrowid is None
-        # issue 2682
-        cur.execute('''insert
-                    into test 
-                    values (?)
-                    ''', (1, ))
-        assert cur.lastrowid is not None
-        cur.execute('''insert\t into test values (?) ''', (1, ))
-        assert cur.lastrowid is not None
-
-    def test_authorizer_bad_value(self, con):
-        def authorizer_cb(action, arg1, arg2, dbname, source):
-            return 42
-        con.set_authorizer(authorizer_cb)
-        with pytest.raises(_sqlite3.OperationalError) as e:
-            con.execute('select 123')
-        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-        if (int(major), int(minor), int(micro)) >= (3, 6, 14):
-            assert str(e.value) == 'authorizer malfunction'
-        else:
-            assert str(e.value) == \
-                ("illegal return value (1) from the authorization function - "
-                 "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
-
-    def test_issue1573(self, con):
-        cur = con.cursor()
-        cur.execute(u'SELECT 1 as m&#233;il')
-        assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
-
-    def test_adapter_exception(self, con):
-        def cast(obj):
-            raise ZeroDivisionError
-
-        _sqlite3.register_adapter(int, cast)
-        try:
-            cur = con.cursor()
-            cur.execute("select ?", (4,))
-            val = cur.fetchone()[0]
-            # Adapter error is ignored, and parameter is passed as is.
-            assert val == 4
-            assert type(val) is int
-        finally:
-            del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
-
-    def test_null_character(self, con):
-        if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9):
-            pytest.skip("_sqlite3 too old")
-        exc = raises(ValueError, con, "\0select 1")
-        assert str(exc.value) == "the query contains a null character"
-        exc = raises(ValueError, con, "select 1\0")
-        assert str(exc.value) == "the query contains a null character"
-        cur = con.cursor()
-        exc = raises(ValueError, cur.execute, "\0select 2")
-        assert str(exc.value) == "the query contains a null character"
-        exc = raises(ValueError, cur.execute, "select 2\0")
-        assert str(exc.value) == "the query contains a null character"
-
-    def test_close_in_del_ordering(self):
-        import gc
-        class SQLiteBackend(object):
-            success = False
-            def __init__(self):
-                self.connection = _sqlite3.connect(":memory:")
-            def close(self):
-                self.connection.close()
-            def __del__(self):
-                self.close()
-                SQLiteBackend.success = True
-            def create_db_if_needed(self):
-                conn = self.connection
-                cursor = conn.cursor()
-                cursor.execute("""
-                    create table if not exists nameoftable(value text)
-                """)
-                cursor.close()
-                conn.commit()
-        SQLiteBackend().create_db_if_needed()
-        gc.collect()
-        gc.collect()
-        assert SQLiteBackend.success
-
-
-class TestSQLiteHost(BaseTestSQLite):
-    def setup_class(cls):
-        global _sqlite3
-        import _sqlite3
-
-
-class TestSQLitePyPy(BaseTestSQLite):
-    def setup_class(cls):
-        if sys.version_info < (2, 7):
-            pytest.skip("_sqlite3 requires Python 2.7")
-
-        try:
-            from lib_pypy import _sqlite3_cffi
-        except ImportError:
-            # On CPython, "pip install cffi".  On old PyPy's, no chance
-            pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py "
-                        "manually first")
-
-        global _sqlite3
-        from lib_pypy import _sqlite3
+def test_close_in_del_ordering():
+    import gc
+    class SQLiteBackend(object):
+        success = False
+        def __init__(self):
+            self.connection = _sqlite3.connect(":memory:")
+        def close(self):
+            self.connection.close()
+        def __del__(self):
+            self.close()
+            SQLiteBackend.success = True
+        def create_db_if_needed(self):
+            conn = self.connection
+            cursor = conn.cursor()
+            cursor.execute("""
+                create table if not exists nameoftable(value text)
+            """)
+            cursor.close()
+            conn.commit()
+    SQLiteBackend().create_db_if_needed()
+    gc.collect()
+    gc.collect()
+    assert SQLiteBackend.success
diff --git a/rpython/jit/backend/x86/runner.py 
b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -44,7 +44,7 @@
             if config.translation.jit_profiler == "oprofile":
                 from rpython.jit.backend.x86 import oprofile
                 if not oprofile.OPROFILE_AVAILABLE:
-                    log.WARNING('oprofile support was explicitly enabled, but 
oprofile headers seem not to be available')
+                    raise Exception('oprofile support was explicitly enabled, 
but oprofile headers seem not to be available')
                 profile_agent = oprofile.OProfileAgent()
             self.with_threads = config.translation.thread
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py 
b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -11,7 +11,7 @@
 from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
-     AbstractResOp, GuardResOp
+     GuardResOp
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.optimizeopt import info
         
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py 
b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, 
CONST_1,
     CONST_0)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, AbstractResOp
+from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.optimizeopt import vstring
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.rlib.rarithmetic import intmask
@@ -44,8 +44,9 @@
         if b.has_lower and b.has_upper and b.lower == b.upper:
             self.make_constant_int(box, b.lower)
 
-        if isinstance(box, AbstractResOp):
-            dispatch_bounds_ops(self, box)
+        box1 = self.optimizer.as_operation(box)
+        if box1 is not None:
+            dispatch_bounds_ops(self, box1)
 
     def _optimize_guard_true_false_value(self, op):
         return self.emit(op)
@@ -126,10 +127,11 @@
             v1, v2 = v2, v1
         # if both are constant, the pure optimization will deal with it
         if v2.is_constant() and not v1.is_constant():
-            if not self.optimizer.is_inputarg(arg1):
+            arg1 = self.optimizer.as_operation(arg1)
+            if arg1 is not None:
                 if arg1.getopnum() == rop.INT_ADD:
-                    prod_arg1 = arg1.getarg(0)
-                    prod_arg2 = arg1.getarg(1)
+                    prod_arg1 = self.get_box_replacement(arg1.getarg(0))
+                    prod_arg2 = self.get_box_replacement(arg1.getarg(1))
                     prod_v1 = self.getintbound(prod_arg1)
                     prod_v2 = self.getintbound(prod_arg2)
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py 
b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -279,6 +279,7 @@
         self.quasi_immutable_deps = None
         self.replaces_guard = {}
         self._newoperations = []
+        self._emittedoperations = {}
         self.optimizer = self
         self.optpure = None
         self.optheap = None
@@ -292,11 +293,6 @@
         self.set_optimizations(optimizations)
         self.setup()
 
-    def init_inparg_dict_from(self, lst):
-        self.inparg_dict = {}
-        for box in lst:
-            self.inparg_dict[box] = None
-
     def set_optimizations(self, optimizations):
         if optimizations:
             self.first_optimization = optimizations[0]
@@ -384,9 +380,12 @@
             return info.force_box(op, optforce)
         return op
 
-    def is_inputarg(self, op):
-        return True
-        return op in self.inparg_dict
+    def as_operation(self, op):
+        # You should never check "isinstance(op, AbstractResOp" directly.
+        # Instead, use this helper.
+        if isinstance(op, AbstractResOp) and op in self._emittedoperations:
+            return op
+        return None
 
     def get_constant_box(self, box):
         box = self.get_box_replacement(box)
@@ -404,6 +403,7 @@
 
     def clear_newoperations(self):
         self._newoperations = []
+        self._emittedoperations = {}
 
     def make_equal_to(self, op, newop):
         op = self.get_box_replacement(op)
@@ -631,6 +631,7 @@
             self._last_guard_op = None
         self._really_emitted_operation = op
         self._newoperations.append(op)
+        self._emittedoperations[op] = None
 
     def emit_guard_operation(self, op, pendingfields):
         guard_op = op # self.replace_op_with(op, op.getopnum())
@@ -675,6 +676,7 @@
             return
         newop = self.replace_op_with_no_ovf(op)
         self._newoperations[-1] = newop
+        self._emittedoperations[newop] = None
 
     def replace_op_with_no_ovf(self, op):
         if op.getopnum() == rop.INT_MUL_OVF:
@@ -719,6 +721,7 @@
         new_descr = new_op.getdescr()
         new_descr.copy_all_attributes_from(old_descr)
         self._newoperations[old_op_pos] = new_op
+        self._emittedoperations[new_op] = None
 
     def store_final_boxes_in_guard(self, op, pendingfields):
         assert pendingfields is not None
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py 
b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -10,7 +10,7 @@
 from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL
 from rpython.jit.metainterp.optimizeopt.util import _findall, 
make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
-     OpHelpers, AbstractResOp
+     OpHelpers
 from rpython.rlib.rarithmetic import highest_bit
 from rpython.rtyper.lltypesystem import llmemory
 from rpython.rtyper import rclass
@@ -389,6 +389,8 @@
     def optimize_GUARD_SUBCLASS(self, op):
         info = self.getptrinfo(op.getarg(0))
         optimizer = self.optimizer
+        # must raise 'InvalidLoop' in all cases where 'info' shows the
+        # class cannot possibly match (see test_issue2926)
         if info and info.is_constant():
             c = self.get_box_replacement(op.getarg(0))
             vtable = optimizer.cpu.ts.cls_of_box(c).getint()
@@ -398,13 +400,29 @@
         if info is not None and info.is_about_object():
             known_class = info.get_known_class(optimizer.cpu)
             if known_class:
+                # Class of 'info' is exactly 'known_class'.
+                # We know statically if the 'guard_subclass' will pass or fail.
                 if optimizer._check_subclass(known_class.getint(),
                                              op.getarg(1).getint()):
                     return
+                else:
+                    raise InvalidLoop(
+                        "GUARD_SUBCLASS(known_class) proven to always fail")
             elif info.get_descr() is not None:
-                if optimizer._check_subclass(info.get_descr().get_vtable(),
+                # Class of 'info' is either get_descr() or a subclass of it.
+                # We're keeping the 'guard_subclass' at runtime only in the
+                # case where get_descr() is some strict parent class of
+                # the argument to 'guard_subclass'.
+                info_base_descr = info.get_descr().get_vtable()
+                if optimizer._check_subclass(info_base_descr,
                                              op.getarg(1).getint()):
-                    return
+                    return    # guard_subclass always passing
+                elif optimizer._check_subclass(op.getarg(1).getint(),
+                                               info_base_descr):
+                    pass      # don't know, must keep the 'guard_subclass'
+                else:
+                    raise InvalidLoop(
+                        "GUARD_SUBCLASS(base_class) proven to always fail")
         return self.emit(op)
 
     def optimize_GUARD_NONNULL(self, op):
@@ -490,11 +508,11 @@
 
     def postprocess_GUARD_TRUE(self, op):
         box = self.get_box_replacement(op.getarg(0))
-        if (isinstance(box, AbstractResOp) and
-                box.getopnum() == rop.INT_IS_TRUE):
+        box1 = self.optimizer.as_operation(box)
+        if box1 is not None and box1.getopnum() == rop.INT_IS_TRUE:
             # we can't use the (current) range analysis for this because
             # "anything but 0" is not a valid range
-            self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0)
+            self.pure_from_args(rop.INT_IS_ZERO, [box1.getarg(0)], CONST_0)
         self.make_constant(box, CONST_1)
 
     def optimize_GUARD_FALSE(self, op):
@@ -502,11 +520,11 @@
 
     def postprocess_GUARD_FALSE(self, op):
         box = self.get_box_replacement(op.getarg(0))
-        if (isinstance(box, AbstractResOp) and
-                box.getopnum() == rop.INT_IS_ZERO):
+        box1 = self.optimizer.as_operation(box)
+        if box1 is not None and box1.getopnum() == rop.INT_IS_ZERO:
             # we can't use the (current) range analysis for this because
             # "anything but 0" is not a valid range
-            self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1)
+            self.pure_from_args(rop.INT_IS_TRUE, [box1.getarg(0)], CONST_1)
         self.make_constant(box, CONST_0)
 
     def optimize_ASSERT_NOT_NONE(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py 
b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4017,7 +4017,6 @@
         strsetitem(p3, i2, i0)
         i5 = int_add(i2, 1)
         strsetitem(p3, i5, i1)
-        ifoo = int_add(i5, 1)
         jump(i1, i0, p3)
         """
         self.optimize_strunicode_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py 
b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -74,7 +74,7 @@
         print "Loop:"
         print '\n'.join([str(o) for o in loop.operations])
         print
-        if expected_short:
+        if expected_short or getattr(info, 'short_preamble', None):
             print "Short Preamble:"
             short = info.short_preamble
             print '\n'.join([str(o) for o in short])
@@ -1300,7 +1300,7 @@
         preamble = """
         [i0, p1, p3]
         i28 = int_add(i0, 1)
-        i29 = int_add(i28, 1)
+        i29 = int_add(i0, 2)
         p30 = new_with_vtable(descr=nodesize)
         setfield_gc(p30, i28, descr=valuedescr)
         setfield_gc(p3, p30, descr=nextdescr)
@@ -1310,7 +1310,7 @@
         expected = """
         [i0, p1, p3]
         i28 = int_add(i0, 1)
-        i29 = int_add(i28, 1)
+        i29 = int_add(i0, 2)
         p30 = new_with_vtable(descr=nodesize)
         setfield_gc(p30, i28, descr=valuedescr)
         setfield_gc(p3, p30, descr=nextdescr)
@@ -6392,7 +6392,6 @@
         strsetitem(p3, i2, i0)
         i5 = int_add(i2, 1)
         strsetitem(p3, i5, i1)
-        i6 = int_add(i5, 1)      # will be killed by the backend
         jump(i1, i0, p3)
         """
         self.optimize_strunicode_loop(ops, expected, expected)
@@ -9063,6 +9062,7 @@
         self.optimize_loop(ops, expected)
 
     def test_same_as_preserves_info_in_the_preamble_2(self):
+        py.test.xfail("less efficient loop, investigate")
         ops = """
         [i0, p0]
         ifoo = getfield_gc_i(p0, descr=valuedescr)
@@ -9499,5 +9499,25 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_issue2904(self):
+        # we don't store advanced virtualstate information like "i1 = i2 + 1",
+        # which means that the following loop, when unrolled, cannot be
+        # optimized based on the knowledge that "i1 = i2 + 1" from the
+        # preamble---we can't use that knowledge.  After the fix, we get
+        # the value "i2 + 1" passed as a third argument, possibly different
+        # from "i1".
+        ops = """
+        [i1, i2]
+        guard_value(i1, 10) []
+        i3 = int_add(i2, 1)
+        jump(i3, i2)
+        """
+        expected = """
+        [i1, i2, i3]
+        guard_value(i1, 10) []
+        jump(i3, i2, i3)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py 
b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -10,8 +10,7 @@
 from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp,\
-     AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
 from rpython.jit.metainterp import compile
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
@@ -22,7 +21,6 @@
             if self.optunroll.short_preamble_producer is None:
                 assert False # unreachable code
             op = preamble_op.op
-            self.optimizer.inparg_dict[op] = None # XXX ARGH
             # special hack for int_add(x, accumulator-const) optimization
             self.optunroll.short_preamble_producer.use_box(op,
                                                 preamble_op.preamble_op, self)
@@ -144,7 +142,6 @@
         except VirtualStatesCantMatch:
             raise InvalidLoop("Cannot import state, virtual states don't 
match")
         self.potential_extra_ops = {}
-        self.optimizer.init_inparg_dict_from(label_args)
         try:
             info, _ = self.optimizer.propagate_all_forward(
                 trace, call_pure_results, flush=False)
@@ -431,8 +428,9 @@
                 for box in self._map_args(mapping, short_jump_args)]
 
     def _expand_info(self, arg, infos):
-        if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum):
-            info = self.optimizer.getinfo(arg.getarg(0))
+        arg1 = self.optimizer.as_operation(arg)
+        if arg1 is not None and rop.is_same_as(arg1.opnum):
+            info = self.optimizer.getinfo(arg1.getarg(0))
         else:
             info = self.optimizer.getinfo(arg)
         if arg in infos:
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py 
b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -6,8 +6,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1
 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.unroll import unrolling_iterable
diff --git a/rpython/jit/metainterp/test/test_ajit.py 
b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4702,3 +4702,112 @@
         res = self.meta_interp(f, [10])
         assert res == f(10)
 
+    def test_cached_info_missing(self):
+        py.test.skip("XXX hitting a non-translated assert in 
optimizeopt/heap.py, but seems not to hurt the rest")
+        driver = JitDriver(greens = [],
+                           reds=['iterations', 'total', 'c', 'height', 'h'])
+
+        class IntVal:
+            _immutable_fields_ = ['intval']
+            def __init__(self, value):
+                self.intval = value
+
+        def f(height, iterations):
+            height = IntVal(height)
+            c = IntVal(0)
+            h = height
+            total = IntVal(0)
+
+            while True:
+                driver.jit_merge_point(iterations=iterations,
+                        total=total, c=c, height=height, h=h)
+                if h.intval > 0:
+                    h = IntVal(h.intval - 1)
+                    total = IntVal(total.intval + 1)
+                else:
+                    c = IntVal(c.intval + 1)
+                    if c.intval >= iterations:
+                        return total.intval
+                    h = height
+
+        res = self.meta_interp(f, [2, 200])
+        assert res == f(2, 200)
+
+    def test_issue2904(self):
+        driver = JitDriver(greens = [],
+                           reds=['iterations', 'total', 'c', 'height', 'h'])
+
+        def f(height, iterations):
+            set_param(driver, 'threshold', 4)
+            set_param(driver, 'trace_eagerness', 1)
+            c = 0
+            h = height
+            total = 0
+
+            while True:
+                driver.jit_merge_point(iterations=iterations,
+                        total=total, c=c, height=height, h=h)
+                if h != 0:
+                    h = h - 1
+                    total = total + 1
+                else:
+                    c = c + 1
+                    if c >= iterations:
+                        return total
+                    h = height - 1
+
+        res = self.meta_interp(f, [2, 200])
+        assert res == f(2, 200)
+
+    def test_issue2926(self):
+        driver = JitDriver(greens = [], reds=['i', 'total', 'p'])
+
+        class Base(object):
+            def do_stuff(self):
+                return 1000
+        class Int(Base):
+            def __init__(self, intval):
+                self.intval = intval
+            def do_stuff(self):
+                return self.intval
+        class SubInt(Int):
+            pass
+        class Float(Base):
+            def __init__(self, floatval):
+                self.floatval = floatval
+            def do_stuff(self):
+                return int(self.floatval)
+
+        prebuilt = [Int(i) for i in range(10)]
+
+        @dont_look_inside
+        def forget_intbounds(i):
+            return i
+
+        @dont_look_inside
+        def escape(p):
+            pass
+
+        def f(i):
+            total = 0
+            p = Base()
+            while True:
+                driver.jit_merge_point(i=i, total=total, p=p)
+                #print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', i
+                if i == 13:
+                    break
+                total += p.do_stuff()
+                j = forget_intbounds(i)
+                if j < 10:        # initial loop
+                    p = prebuilt[i]
+                    p.intval = j
+                elif j < 12:
+                    p = Int(i)
+                else:
+                    p = Float(3.14)
+                    escape(p)
+                i += 1
+            return total
+
+        res = self.meta_interp(f, [0])
+        assert res == f(0)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -62,6 +62,7 @@
 # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w)
 import sys
 import os
+import time
 from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage
@@ -73,7 +74,6 @@
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import rgc
-from rpython.rlib.rtimer import read_timestamp
 from rpython.memory.gc.minimarkpage import out_of_memory
 
 #
@@ -192,6 +192,8 @@
 
 # ____________________________________________________________
 
+
+
 class IncrementalMiniMarkGC(MovingGCBase):
     _alloc_flavor_ = "raw"
     inline_simple_malloc = True
@@ -374,6 +376,7 @@
         self.raw_malloc_might_sweep = self.AddressStack()
         self.rawmalloced_total_size = r_uint(0)
         self.rawmalloced_peak_size = r_uint(0)
+        self.total_gc_time = 0.0
 
         self.gc_state = STATE_SCANNING
         #
@@ -1644,7 +1647,7 @@
         """Perform a minor collection: find the objects from the nursery
         that remain alive and move them out."""
         #
-        start = read_timestamp()
+        start = time.time()
         debug_start("gc-minor")
         #
         # All nursery barriers are invalid from this point on.  They
@@ -1843,7 +1846,8 @@
         self.root_walker.finished_minor_collection()
         #
         debug_stop("gc-minor")
-        duration = read_timestamp() - start
+        duration = time.time() - start
+        self.total_gc_time += duration
         self.hooks.fire_gc_minor(
             duration=duration,
             total_memory_used=total_memory_used,
@@ -2249,7 +2253,7 @@
     # Note - minor collections seem fast enough so that one
     # is done before every major collection step
     def major_collection_step(self, reserving_size=0):
-        start = read_timestamp()
+        start = time.time()
         debug_start("gc-collect-step")
         oldstate = self.gc_state
         debug_print("starting gc state: ", GC_STATES[self.gc_state])
@@ -2493,7 +2497,8 @@
 
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
-        duration = read_timestamp() - start
+        duration = time.time() - start
+        self.total_gc_time += duration
         self.hooks.fire_gc_collect_step(
             duration=duration,
             oldstate=oldstate,
@@ -3000,6 +3005,8 @@
                                self.ac.total_memory_used))
         elif stats_no == rgc.NURSERY_SIZE:
             return intmask(self.nursery_size)
+        elif stats_no == rgc.TOTAL_GC_TIME:
+            return int(self.total_gc_time * 1000)
         return 0
 
 
diff --git a/rpython/memory/gc/test/test_hook.py 
b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -70,7 +70,7 @@
         assert self.gc.hooks.minors == [
             {'total_memory_used': 0, 'pinned_objects': 0}
             ]
-        assert self.gc.hooks.durations[0] > 0
+        assert self.gc.hooks.durations[0] > 0.
         self.gc.hooks.reset()
         #
         # these objects survive, so the total_memory_used is > 0
@@ -103,7 +103,7 @@
             ]
         assert len(self.gc.hooks.durations) == 4 # 4 steps
         for d in self.gc.hooks.durations:
-            assert d > 0
+            assert d > 0.0
         self.gc.hooks.reset()
         #
         self.stackroots.append(self.malloc(S))
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -1,6 +1,7 @@
 import sys
 import time
 
+from rpython.rlib.objectmodel import enforceargs
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rlib.objectmodel import we_are_translated, always_inline
 from rpython.rlib.rarithmetic import is_valid_int, r_longlong
@@ -75,6 +76,7 @@
     _stop_colors = ""
 
 @always_inline
+@enforceargs(str, bool)
 def debug_start(category, timestamp=False):
     """
     Start a PYPYLOG section.
@@ -85,6 +87,7 @@
     return _debug_start(category, timestamp)
 
 @always_inline
+@enforceargs(str, bool)
 def debug_stop(category, timestamp=False):
     """
     Stop a PYPYLOG section. See debug_start for docs about timestamp
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -704,7 +704,7 @@
 (TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE,
  PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY,
  TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY,
- NURSERY_SIZE) = range(10)
+ NURSERY_SIZE, TOTAL_GC_TIME) = range(11)
 
 @not_rpython
 def get_stats(stat_no):
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -136,7 +136,10 @@
     void = lltype.nullptr(rffi.VOIDP.TO)
     result = -1.0
     if HAVE_GETTIMEOFDAY:
-        with lltype.scoped_alloc(TIMEVAL) as t:
+        # NB: can't use lltype.scoped_malloc, because that will allocate the
+        # with handler in the GC, but we want to use time.time from gc.collect!
+        t = lltype.malloc(TIMEVAL, flavor='raw')
+        try:
             errcode = -1
             if GETTIMEOFDAY_NO_TZ:
                 errcode = c_gettimeofday(t)
@@ -145,13 +148,18 @@
 
             if rffi.cast(rffi.LONG, errcode) == 0:
                 result = decode_timeval(t)
+        finally:
+            lltype.free(t, flavor='raw')
         if result != -1:
             return result
     else: # assume using ftime(3)
-        with lltype.scoped_alloc(TIMEB) as t:
+        t = lltype.malloc(TIMEB, flavor='raw')
+        try:
             c_ftime(t)
             result = (float(intmask(t.c_time)) +
                       float(intmask(t.c_millitm)) * 0.001)
+        finally:
+            lltype.free(t, flavor='raw')
         return result
     return float(c_time(void))
 
diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py 
b/rpython/translator/backendopt/test/test_mallocprediction.py
--- a/rpython/translator/backendopt/test/test_mallocprediction.py
+++ b/rpython/translator/backendopt/test/test_mallocprediction.py
@@ -179,7 +179,7 @@
     t, graph = rtype(entry_point, [int])
     total0 = preparation(t, t.graphs)
     total = clever_inlining_and_malloc_removal(t)
-    assert total0 + total == 10
+    assert total0 + total == 9
 
 def test_loop():
     l = [10, 12, 15, 1]
diff --git a/rpython/translator/c/test/test_newgc.py 
b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1812,7 +1812,20 @@
         res = self.run("ignore_finalizer")
         assert res == 1    # translated: x1 is removed from the list
 
+    def define_total_gc_time(cls):
+        def f():
+            l = []
+            for i in range(1000000):
+                l.append(str(i))
+            l = []
+            for i in range(10):
+                rgc.collect()
+            return rgc.get_stats(rgc.TOTAL_GC_TIME)
+        return f
 
+    def test_total_gc_time(self):
+        res = self.run("total_gc_time")
+        assert res > 0 # should take a few microseconds
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):
diff --git a/rpython/translator/c/test/test_standalone.py 
b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -521,11 +521,9 @@
         assert path.check(file=0)
 
     def test_debug_start_stop_timestamp(self):
-        import sys
-        import time
         from rpython.rlib.rtimer import read_timestamp
         def entry_point(argv):
-            timestamp = int(argv[1])
+            timestamp = bool(int(argv[1]))
             ts1 = debug_start("foo", timestamp=timestamp)
             ts2 = read_timestamp()
             ts3 = debug_stop("foo", timestamp=timestamp)
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

Reply via email to