Author: Matti Picus <[email protected]>
Branch: unicode-utf8-py3
Changeset: r95449:42c5ce5c6072
Date: 2018-12-09 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/42c5ce5c6072/
Log: merge py3.5 into branch
diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py
b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py
--- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py
+++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py
@@ -4,7 +4,12 @@
from cffi import cffi_opcode
if '__pypy__' in sys.builtin_module_names:
- py.test.skip("not available on pypy", allow_module_level=True)
+ try:
+ # pytest >= 4.0
+ py.test.skip("not available on pypy", allow_module_level=True)
+ except TypeError:
+ # older pytest
+ py.test.skip("not available on pypy")
cffi_dir = os.path.dirname(cffi_opcode.__file__)
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -4,8 +4,10 @@
from errno import EINVAL, EPERM
import _structseq, os
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+try:
+ from __pypy__ import builtinify
+except ImportError:
+ builtinify = lambda f: f
class error(Exception):
@@ -35,7 +37,7 @@
ru_oublock = _structseq.structseqfield(10, "block output operations")
ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent")
ru_msgrcv = _structseq.structseqfield(12, "IPC messages received")
- ru_nsignals = _structseq.structseqfield(13,"signals received")
+ ru_nsignals = _structseq.structseqfield(13, "signals received")
ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches")
ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches")
@@ -57,7 +59,7 @@
ru.ru_nsignals,
ru.ru_nvcsw,
ru.ru_nivcsw,
- ))
+ ))
@builtinify
def getrusage(who):
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -58,4 +58,9 @@
.. branch: rlock-in-rpython
-Backport CPython fix for `thread.RLock`
\ No newline at end of file
+Backport CPython fix for `thread.RLock`
+
+
+.. branch: expose-gc-time
+
+Make GC hooks measure time in seconds (as opposed to an opaque unit).
\ No newline at end of file
diff --git a/pypy/goal/targetpypystandalone.py
b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -386,7 +386,7 @@
def get_gchooks(self):
from pypy.module.gc.hook import LowLevelGcHooks
if self.space is None:
- raise Exception("get_gchooks must be called afeter
get_entry_point")
+ raise Exception("get_gchooks must be called after get_entry_point")
return self.space.fromcache(LowLevelGcHooks)
def get_entry_point(self, config):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -603,8 +603,14 @@
warnoptions = pythonwarnings.split(',') + warnoptions
if warnoptions:
sys.warnoptions[:] = warnoptions
- from warnings import _processoptions
- _processoptions(sys.warnoptions)
+ try:
+ if 'warnings' in sys.modules:
+ from warnings import _processoptions
+ _processoptions(sys.warnoptions)
+ else:
+ import warnings
+ except ImportError as e:
+ pass # CPython just eats any exception here
# set up the Ctrl-C => KeyboardInterrupt signal handler, if the
# signal module is available
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -57,12 +57,14 @@
'total_allocated_memory', 'jit_backend_allocated',
'peak_memory', 'peak_allocated_memory',
'total_arena_memory',
'total_rawmalloced_memory', 'nursery_size',
- 'peak_arena_memory', 'peak_rawmalloced_memory'):
+ 'peak_arena_memory', 'peak_rawmalloced_memory',
+ ):
setattr(self, item, self._format(getattr(self._s, item)))
self.memory_used_sum = self._format(self._s.total_gc_memory +
self._s.total_memory_pressure +
self._s.jit_backend_used)
self.memory_allocated_sum =
self._format(self._s.total_allocated_memory + self._s.total_memory_pressure +
self._s.jit_backend_allocated)
+ self.total_gc_time = self._s.total_gc_time
def _format(self, v):
if v < 1000000:
@@ -92,6 +94,8 @@
raw assembler allocated: %s%s
-----------------------------
Total: %s
+
+ Total time spent in GC: %s
""" % (self.total_gc_memory, self.peak_memory,
self.total_arena_memory,
self.total_rawmalloced_memory,
@@ -106,7 +110,8 @@
self.nursery_size,
self.jit_backend_allocated,
extra,
- self.memory_allocated_sum)
+ self.memory_allocated_sum,
+ self.total_gc_time / 1000.0)
def get_stats(memory_pressure=False):
diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -7,6 +7,8 @@
from pypy.interpreter.typedef import TypeDef, interp_attrproperty,
GetSetProperty
from pypy.interpreter.executioncontext import AsyncAction
+inf = float("inf")
+
class LowLevelGcHooks(GcHooks):
"""
These are the low-level hooks which are called directly from the GC.
@@ -126,9 +128,9 @@
def reset(self):
self.count = 0
- self.duration = r_longlong(0)
- self.duration_min = r_longlong(longlongmax)
- self.duration_max = r_longlong(0)
+ self.duration = 0.0
+ self.duration_min = inf
+ self.duration_max = 0.0
def fix_annotation(self):
# the annotation of the class and its attributes must be completed
@@ -136,9 +138,9 @@
# annotated with the correct types
if NonConstant(False):
self.count = NonConstant(-42)
- self.duration = NonConstant(r_longlong(-42))
- self.duration_min = NonConstant(r_longlong(-42))
- self.duration_max = NonConstant(r_longlong(-42))
+ self.duration = NonConstant(-53.2)
+ self.duration_min = NonConstant(-53.2)
+ self.duration_max = NonConstant(-53.2)
self.total_memory_used = NonConstant(r_uint(42))
self.pinned_objects = NonConstant(-42)
self.fire()
@@ -166,9 +168,9 @@
def reset(self):
self.count = 0
- self.duration = r_longlong(0)
- self.duration_min = r_longlong(longlongmax)
- self.duration_max = r_longlong(0)
+ self.duration = 0.0
+ self.duration_min = inf
+ self.duration_max = 0.0
def fix_annotation(self):
# the annotation of the class and its attributes must be completed
@@ -176,9 +178,9 @@
# annotated with the correct types
if NonConstant(False):
self.count = NonConstant(-42)
- self.duration = NonConstant(r_longlong(-42))
- self.duration_min = NonConstant(r_longlong(-42))
- self.duration_max = NonConstant(r_longlong(-42))
+ self.duration = NonConstant(-53.2)
+ self.duration_min = NonConstant(-53.2)
+ self.duration_max = NonConstant(-53.2)
self.oldstate = NonConstant(-42)
self.newstate = NonConstant(-42)
self.fire()
@@ -276,10 +278,14 @@
# just a shortcut to make the typedefs shorter
-def wrap_many_ints(cls, names):
+def wrap_many(cls, names):
d = {}
for name in names:
- d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint")
+ if "duration" in name:
+ wrapfn = "newfloat"
+ else:
+ wrapfn = "newint"
+ d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn)
return d
@@ -303,7 +309,7 @@
W_GcMinorStats.typedef = TypeDef(
"GcMinorStats",
- **wrap_many_ints(W_GcMinorStats, (
+ **wrap_many(W_GcMinorStats, (
"count",
"duration",
"duration_min",
@@ -319,7 +325,7 @@
STATE_SWEEPING = incminimark.STATE_SWEEPING,
STATE_FINALIZING = incminimark.STATE_FINALIZING,
GC_STATES = tuple(incminimark.GC_STATES),
- **wrap_many_ints(W_GcCollectStepStats, (
+ **wrap_many(W_GcCollectStepStats, (
"count",
"duration",
"duration_min",
@@ -330,7 +336,7 @@
W_GcCollectStats.typedef = TypeDef(
"GcCollectStats",
- **wrap_many_ints(W_GcCollectStats, (
+ **wrap_many(W_GcCollectStats, (
"count",
"num_major_collects",
"arenas_count_before",
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -189,6 +189,7 @@
self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY)
self.peak_rawmalloced_memory =
rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY)
self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE)
+ self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME)
W_GcStats.typedef = TypeDef("GcStats",
total_memory_pressure=interp_attrproperty("total_memory_pressure",
@@ -215,6 +216,8 @@
cls=W_GcStats, wrapfn="newint"),
nursery_size=interp_attrproperty("nursery_size",
cls=W_GcStats, wrapfn="newint"),
+ total_gc_time=interp_attrproperty("total_gc_time",
+ cls=W_GcStats, wrapfn="newint"),
)
@unwrap_spec(memory_pressure=bool)
diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py
--- a/pypy/module/gc/test/test_hook.py
+++ b/pypy/module/gc/test/test_hook.py
@@ -26,11 +26,11 @@
@unwrap_spec(ObjSpace)
def fire_many(space):
- gchooks.fire_gc_minor(5, 0, 0)
- gchooks.fire_gc_minor(7, 0, 0)
- gchooks.fire_gc_collect_step(5, 0, 0)
- gchooks.fire_gc_collect_step(15, 0, 0)
- gchooks.fire_gc_collect_step(22, 0, 0)
+ gchooks.fire_gc_minor(5.0, 0, 0)
+ gchooks.fire_gc_minor(7.0, 0, 0)
+ gchooks.fire_gc_collect_step(5.0, 0, 0)
+ gchooks.fire_gc_collect_step(15.0, 0, 0)
+ gchooks.fire_gc_collect_step(22.0, 0, 0)
gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6)
cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor))
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py
b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -5,327 +5,321 @@
import pytest
import sys
+_sqlite3 = pytest.importorskip('_sqlite3')
-def pytest_funcarg__con(request):
+pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
+ reason="PyPy-only test")
+
+
[email protected]_fixture
+def con():
con = _sqlite3.connect(':memory:')
- request.addfinalizer(lambda: con.close())
- return con
+ yield con
+ con.close()
-class BaseTestSQLite:
- def test_list_ddl(self, con):
- """From issue996. Mostly just looking for lack of exceptions."""
- cursor = con.cursor()
- cursor.execute('CREATE TABLE foo (bar INTEGER)')
- result = list(cursor)
- assert result == []
- cursor.execute('INSERT INTO foo (bar) VALUES (42)')
- result = list(cursor)
- assert result == []
- cursor.execute('SELECT * FROM foo')
- result = list(cursor)
- assert result == [(42,)]
+def test_list_ddl(con):
+ """From issue996. Mostly just looking for lack of exceptions."""
+ cursor = con.cursor()
+ cursor.execute('CREATE TABLE foo (bar INTEGER)')
+ result = list(cursor)
+ assert result == []
+ cursor.execute('INSERT INTO foo (bar) VALUES (42)')
+ result = list(cursor)
+ assert result == []
+ cursor.execute('SELECT * FROM foo')
+ result = list(cursor)
+ assert result == [(42,)]
- def test_connect_takes_same_positional_args_as_Connection(self, con):
- if not hasattr(_sqlite3, '_ffi'):
- pytest.skip("only works for lib_pypy _sqlite3")
- from inspect import getargspec
- clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore
self
- conargs = getargspec(_sqlite3.connect).args
- assert clsargs == conargs
+@pypy_only
+def test_connect_takes_same_positional_args_as_Connection(con):
+ from inspect import getargspec
+ clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self
+ conargs = getargspec(_sqlite3.connect).args
+ assert clsargs == conargs
- def test_total_changes_after_close(self, con):
- con.close()
- pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+def test_total_changes_after_close(con):
+ con.close()
+ with pytest.raises(_sqlite3.ProgrammingError):
+ con.total_changes
- def test_connection_check_init(self):
- class Connection(_sqlite3.Connection):
- def __init__(self, name):
- pass
+def test_connection_check_init():
+ class Connection(_sqlite3.Connection):
+ def __init__(self, name):
+ pass
- con = Connection(":memory:")
- e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
- assert '__init__' in str(e.value)
+ con = Connection(":memory:")
+ with pytest.raises(_sqlite3.ProgrammingError) as excinfo:
+ con.cursor()
+ assert '__init__' in str(excinfo.value)
- def test_cursor_check_init(self, con):
- class Cursor(_sqlite3.Cursor):
- def __init__(self, name):
- pass
- cur = Cursor(con)
- e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
- assert '__init__' in str(e.value)
+def test_cursor_check_init(con):
+ class Cursor(_sqlite3.Cursor):
+ def __init__(self, name):
+ pass
- def test_connection_after_close(self, con):
- pytest.raises(TypeError, "con()")
- con.close()
- # raises ProgrammingError because should check closed before check args
- pytest.raises(_sqlite3.ProgrammingError, "con()")
+ cur = Cursor(con)
+ with pytest.raises(_sqlite3.ProgrammingError) as excinfo:
+ cur.execute('select 1')
+ assert '__init__' in str(excinfo.value)
- def test_cursor_iter(self, con):
+def test_connection_after_close(con):
+ with pytest.raises(TypeError):
+ con()
+ con.close()
+ # raises ProgrammingError because should check closed before check args
+ with pytest.raises(_sqlite3.ProgrammingError):
+ con()
+
+def test_cursor_iter(con):
+ cur = con.cursor()
+ with pytest.raises(StopIteration):
+ next(cur)
+
+ cur.execute('select 1')
+ next(cur)
+ with pytest.raises(StopIteration):
+ next(cur)
+
+ cur.execute('select 1')
+ con.commit()
+ next(cur)
+ with pytest.raises(StopIteration):
+ next(cur)
+
+ with pytest.raises(_sqlite3.ProgrammingError):
+ cur.executemany('select 1', [])
+ with pytest.raises(StopIteration):
+ next(cur)
+
+ cur.execute('select 1')
+ cur.execute('create table test(ing)')
+ with pytest.raises(StopIteration):
+ next(cur)
+
+ cur.execute('select 1')
+ cur.execute('insert into test values(1)')
+ con.commit()
+ with pytest.raises(StopIteration):
+ next(cur)
+
+def test_cursor_after_close(con):
+ cur = con.execute('select 1')
+ cur.close()
+ con.close()
+ with pytest.raises(_sqlite3.ProgrammingError):
+ cur.close()
+ # raises ProgrammingError because should check closed before check args
+ with pytest.raises(_sqlite3.ProgrammingError):
+ cur.execute(1,2,3,4,5)
+ with pytest.raises(_sqlite3.ProgrammingError):
+ cur.executemany(1,2,3,4,5)
+
+@pypy_only
+def test_connection_del(tmpdir):
+ """For issue1325."""
+ import os
+ import gc
+ resource = pytest.importorskip('resource')
+
+ limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+ try:
+ fds = 0
+ while True:
+ fds += 1
+ resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
+ try:
+ for p in os.pipe(): os.close(p)
+ except OSError:
+ assert fds < 100
+ else:
+ break
+
+ def open_many(cleanup):
+ con = []
+ for i in range(3):
+ con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
+ if cleanup:
+ con[i] = None
+ gc.collect(); gc.collect()
+
+ with pytest.raises(_sqlite3.OperationalError):
+ open_many(False)
+ sys.exc_clear()
+ gc.collect(); gc.collect()
+ open_many(True)
+ finally:
+ resource.setrlimit(resource.RLIMIT_NOFILE, limit)
+
+def test_on_conflict_rollback_executemany(con):
+ major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+ if (int(major), int(minor), int(micro)) < (3, 2, 2):
+ pytest.skip("requires sqlite3 version >= 3.2.2")
+ con.execute("create table foo(x, unique(x) on conflict rollback)")
+ con.execute("insert into foo(x) values (1)")
+ try:
+ con.executemany("insert into foo(x) values (?)", [[1]])
+ except _sqlite3.DatabaseError:
+ pass
+ con.execute("insert into foo(x) values (2)")
+ try:
+ con.commit()
+ except _sqlite3.OperationalError:
+ pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
+
+def test_statement_arg_checking(con):
+ with pytest.raises(_sqlite3.Warning) as e:
+ con(123)
+ assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
+ with pytest.raises(ValueError) as e:
+ con.execute(123)
+ assert str(e.value) == 'operation parameter must be str or unicode'
+ with pytest.raises(ValueError) as e:
+ con.executemany(123, 123)
+ assert str(e.value) == 'operation parameter must be str or unicode'
+ with pytest.raises(ValueError) as e:
+ con.executescript(123)
+ assert str(e.value) == 'script argument must be unicode or string.'
+
+def test_statement_param_checking(con):
+ con.execute('create table foo(x)')
+ con.execute('insert into foo(x) values (?)', [2])
+ con.execute('insert into foo(x) values (?)', (2,))
+ class seq(object):
+ def __len__(self):
+ return 1
+ def __getitem__(self, key):
+ return 2
+ con.execute('insert into foo(x) values (?)', seq())
+ del seq.__len__
+ with pytest.raises(_sqlite3.ProgrammingError):
+ con.execute('insert into foo(x) values (?)', seq())
+ with pytest.raises(_sqlite3.ProgrammingError):
+ con.execute('insert into foo(x) values (?)', {2:2})
+ with pytest.raises(ValueError) as e:
+ con.execute('insert into foo(x) values (?)', 2)
+ assert str(e.value) == 'parameters are of unsupported type'
+
+def test_explicit_begin(con):
+ con.execute('BEGIN')
+ con.execute('BEGIN ')
+ con.execute('BEGIN')
+ con.commit()
+ con.execute('BEGIN')
+ con.commit()
+
+def test_row_factory_use(con):
+ con.row_factory = 42
+ con.execute('select 1')
+
+def test_returning_blob_must_own_memory(con):
+ import gc
+ con.create_function("returnblob", 0, lambda: buffer("blob"))
+ cur = con.execute("select returnblob()")
+ val = cur.fetchone()[0]
+ for i in range(5):
+ gc.collect()
+ got = (val[0], val[1], val[2], val[3])
+ assert got == ('b', 'l', 'o', 'b')
+ # in theory 'val' should be a read-write buffer
+ # but it's not right now
+ if not hasattr(_sqlite3, '_ffi'):
+ val[1] = 'X'
+ got = (val[0], val[1], val[2], val[3])
+ assert got == ('b', 'X', 'o', 'b')
+
+def test_description_after_fetchall(con):
+ cur = con.cursor()
+ assert cur.description is None
+ cur.execute("select 42").fetchall()
+ assert cur.description is not None
+
+def test_executemany_lastrowid(con):
+ cur = con.cursor()
+ cur.execute("create table test(a)")
+ cur.executemany("insert into test values (?)", [[1], [2], [3]])
+ assert cur.lastrowid is None
+ # issue 2682
+ cur.execute('''insert
+ into test
+ values (?)
+ ''', (1, ))
+ assert cur.lastrowid is not None
+ cur.execute('''insert\t into test values (?) ''', (1, ))
+ assert cur.lastrowid is not None
+
+def test_authorizer_bad_value(con):
+ def authorizer_cb(action, arg1, arg2, dbname, source):
+ return 42
+ con.set_authorizer(authorizer_cb)
+ with pytest.raises(_sqlite3.OperationalError) as e:
+ con.execute('select 123')
+ major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+ if (int(major), int(minor), int(micro)) >= (3, 6, 14):
+ assert str(e.value) == 'authorizer malfunction'
+ else:
+ assert str(e.value) == \
+ ("illegal return value (1) from the authorization function - "
+ "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
+
+def test_issue1573(con):
+ cur = con.cursor()
+ cur.execute(u'SELECT 1 as méil')
+ assert cur.description[0][0] == u"méil".encode('utf-8')
+
+def test_adapter_exception(con):
+ def cast(obj):
+ raise ZeroDivisionError
+
+ _sqlite3.register_adapter(int, cast)
+ try:
cur = con.cursor()
- with pytest.raises(StopIteration):
- next(cur)
+ cur.execute("select ?", (4,))
+ val = cur.fetchone()[0]
+ # Adapter error is ignored, and parameter is passed as is.
+ assert val == 4
+ assert type(val) is int
+ finally:
+ del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
- cur.execute('select 1')
- next(cur)
- with pytest.raises(StopIteration):
- next(cur)
+def test_null_character(con):
+ if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9):
+ pytest.skip("_sqlite3 too old")
+ with raises(ValueError) as excinfo:
+ con("\0select 1")
+ assert str(excinfo.value) == "the query contains a null character"
+ with raises(ValueError) as excinfo:
+ con("select 1\0")
+ assert str(excinfo.value) == "the query contains a null character"
+ cur = con.cursor()
+ with raises(ValueError) as excinfo:
+ cur.execute("\0select 2")
+ assert str(excinfo.value) == "the query contains a null character"
+ with raises(ValueError) as excinfo:
+ cur.execute("select 2\0")
+ assert str(excinfo.value) == "the query contains a null character"
- cur.execute('select 1')
- con.commit()
- next(cur)
- with pytest.raises(StopIteration):
- next(cur)
-
- with pytest.raises(_sqlite3.ProgrammingError):
- cur.executemany('select 1', [])
- with pytest.raises(StopIteration):
- next(cur)
-
- cur.execute('select 1')
- cur.execute('create table test(ing)')
- with pytest.raises(StopIteration):
- next(cur)
-
- cur.execute('select 1')
- cur.execute('insert into test values(1)')
- con.commit()
- with pytest.raises(StopIteration):
- next(cur)
-
- def test_cursor_after_close(self, con):
- cur = con.execute('select 1')
- cur.close()
- con.close()
- pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
- # raises ProgrammingError because should check closed before check args
- pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
- pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
-
- @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
- def test_connection_del(self, tmpdir):
- """For issue1325."""
- import os
- import gc
- try:
- import resource
- except ImportError:
- pytest.skip("needs resource module")
-
- limit = resource.getrlimit(resource.RLIMIT_NOFILE)
- try:
- fds = 0
- while True:
- fds += 1
- resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
- try:
- for p in os.pipe(): os.close(p)
- except OSError:
- assert fds < 100
- else:
- break
-
- def open_many(cleanup):
- con = []
- for i in range(3):
- con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
- if cleanup:
- con[i] = None
- gc.collect(); gc.collect()
-
- pytest.raises(_sqlite3.OperationalError, open_many, False)
- gc.collect(); gc.collect()
- open_many(True)
- finally:
- resource.setrlimit(resource.RLIMIT_NOFILE, limit)
-
- def test_on_conflict_rollback_executemany(self, con):
- major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
- if (int(major), int(minor), int(micro)) < (3, 2, 2):
- pytest.skip("requires sqlite3 version >= 3.2.2")
- con.execute("create table foo(x, unique(x) on conflict rollback)")
- con.execute("insert into foo(x) values (1)")
- try:
- con.executemany("insert into foo(x) values (?)", [[1]])
- except _sqlite3.DatabaseError:
- pass
- con.execute("insert into foo(x) values (2)")
- try:
- con.commit()
- except _sqlite3.OperationalError:
- pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
-
- def test_statement_arg_checking(self, con):
- with pytest.raises(_sqlite3.Warning) as e:
- con(123)
- assert str(e.value) == 'SQL is of wrong type. Must be string or
unicode.'
- with pytest.raises(ValueError) as e:
- con.execute(123)
- assert str(e.value) == 'operation parameter must be str or unicode'
- with pytest.raises(ValueError) as e:
- con.executemany(123, 123)
- assert str(e.value) == 'operation parameter must be str or unicode'
- with pytest.raises(ValueError) as e:
- con.executescript(123)
- assert str(e.value) == 'script argument must be unicode or string.'
-
- def test_statement_param_checking(self, con):
- con.execute('create table foo(x)')
- con.execute('insert into foo(x) values (?)', [2])
- con.execute('insert into foo(x) values (?)', (2,))
- class seq(object):
- def __len__(self):
- return 1
- def __getitem__(self, key):
- return 2
- con.execute('insert into foo(x) values (?)', seq())
- del seq.__len__
- with pytest.raises(_sqlite3.ProgrammingError):
- con.execute('insert into foo(x) values (?)', seq())
- with pytest.raises(_sqlite3.ProgrammingError):
- con.execute('insert into foo(x) values (?)', {2:2})
- with pytest.raises(ValueError) as e:
- con.execute('insert into foo(x) values (?)', 2)
- assert str(e.value) == 'parameters are of unsupported type'
-
- def test_explicit_begin(self, con):
- con.execute('BEGIN')
- con.execute('BEGIN ')
- con.execute('BEGIN')
- con.commit()
- con.execute('BEGIN')
- con.commit()
-
- def test_row_factory_use(self, con):
- con.row_factory = 42
- con.execute('select 1')
-
- def test_returning_blob_must_own_memory(self, con):
- import gc
- con.create_function("returnblob", 0, lambda: buffer("blob"))
- cur = con.execute("select returnblob()")
- val = cur.fetchone()[0]
- for i in range(5):
- gc.collect()
- got = (val[0], val[1], val[2], val[3])
- assert got == ('b', 'l', 'o', 'b')
- # in theory 'val' should be a read-write buffer
- # but it's not right now
- if not hasattr(_sqlite3, '_ffi'):
- val[1] = 'X'
- got = (val[0], val[1], val[2], val[3])
- assert got == ('b', 'X', 'o', 'b')
-
- def test_description_after_fetchall(self, con):
- cur = con.cursor()
- assert cur.description is None
- cur.execute("select 42").fetchall()
- assert cur.description is not None
-
- def test_executemany_lastrowid(self, con):
- cur = con.cursor()
- cur.execute("create table test(a)")
- cur.executemany("insert into test values (?)", [[1], [2], [3]])
- assert cur.lastrowid is None
- # issue 2682
- cur.execute('''insert
- into test
- values (?)
- ''', (1, ))
- assert cur.lastrowid is not None
- cur.execute('''insert\t into test values (?) ''', (1, ))
- assert cur.lastrowid is not None
-
- def test_authorizer_bad_value(self, con):
- def authorizer_cb(action, arg1, arg2, dbname, source):
- return 42
- con.set_authorizer(authorizer_cb)
- with pytest.raises(_sqlite3.OperationalError) as e:
- con.execute('select 123')
- major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
- if (int(major), int(minor), int(micro)) >= (3, 6, 14):
- assert str(e.value) == 'authorizer malfunction'
- else:
- assert str(e.value) == \
- ("illegal return value (1) from the authorization function - "
- "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
-
- def test_issue1573(self, con):
- cur = con.cursor()
- cur.execute(u'SELECT 1 as méil')
- assert cur.description[0][0] == u"méil".encode('utf-8')
-
- def test_adapter_exception(self, con):
- def cast(obj):
- raise ZeroDivisionError
-
- _sqlite3.register_adapter(int, cast)
- try:
- cur = con.cursor()
- cur.execute("select ?", (4,))
- val = cur.fetchone()[0]
- # Adapter error is ignored, and parameter is passed as is.
- assert val == 4
- assert type(val) is int
- finally:
- del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
-
- def test_null_character(self, con):
- if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9):
- pytest.skip("_sqlite3 too old")
- exc = pytest.raises(ValueError, con, "\0select 1")
- assert str(exc.value) == "the query contains a null character"
- exc = pytest.raises(ValueError, con, "select 1\0")
- assert str(exc.value) == "the query contains a null character"
- cur = con.cursor()
- exc = pytest.raises(ValueError, cur.execute, "\0select 2")
- assert str(exc.value) == "the query contains a null character"
- exc = pytest.raises(ValueError, cur.execute, "select 2\0")
- assert str(exc.value) == "the query contains a null character"
-
- def test_close_in_del_ordering(self):
- import gc
- class SQLiteBackend(object):
- success = False
- def __init__(self):
- self.connection = _sqlite3.connect(":memory:")
- def close(self):
- self.connection.close()
- def __del__(self):
- self.close()
- SQLiteBackend.success = True
- def create_db_if_needed(self):
- conn = self.connection
- cursor = conn.cursor()
- cursor.execute("""
- create table if not exists nameoftable(value text)
- """)
- cursor.close()
- conn.commit()
- SQLiteBackend().create_db_if_needed()
- gc.collect()
- gc.collect()
- assert SQLiteBackend.success
-
-
-class TestSQLiteHost(BaseTestSQLite):
- def setup_class(cls):
- global _sqlite3
- import _sqlite3
-
-
-class TestSQLitePyPy(BaseTestSQLite):
- def setup_class(cls):
- if sys.version_info < (2, 7):
- pytest.skip("_sqlite3 requires Python 2.7")
-
- try:
- from lib_pypy import _sqlite3_cffi
- except ImportError:
- # On CPython, "pip install cffi". On old PyPy's, no chance
- pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py "
- "manually first")
-
- global _sqlite3
- from lib_pypy import _sqlite3
+def test_close_in_del_ordering():
+ import gc
+ class SQLiteBackend(object):
+ success = False
+ def __init__(self):
+ self.connection = _sqlite3.connect(":memory:")
+ def close(self):
+ self.connection.close()
+ def __del__(self):
+ self.close()
+ SQLiteBackend.success = True
+ def create_db_if_needed(self):
+ conn = self.connection
+ cursor = conn.cursor()
+ cursor.execute("""
+ create table if not exists nameoftable(value text)
+ """)
+ cursor.close()
+ conn.commit()
+ SQLiteBackend().create_db_if_needed()
+ gc.collect()
+ gc.collect()
+ assert SQLiteBackend.success
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -97,8 +97,8 @@
w_obj = space.call_function(w_inttype, w_obj)
return w_obj
- @unwrap_spec(nbytes=int, byteorder='text', signed=bool)
- def descr_to_bytes(self, space, nbytes, byteorder, signed=False):
+ @unwrap_spec(length=int, byteorder='text', signed=bool)
+ def descr_to_bytes(self, space, length, byteorder, signed=False):
"""to_bytes(...)
int.to_bytes(length, byteorder, *, signed=False) -> bytes
@@ -121,7 +121,7 @@
"""
bigint = space.bigint_w(self)
try:
- byte_string = bigint.tobytes(nbytes, byteorder=byteorder,
+ byte_string = bigint.tobytes(length, byteorder=byteorder,
signed=signed)
except InvalidEndiannessError:
raise oefmt(space.w_ValueError,
diff --git a/pypy/objspace/std/test/test_longobject.py
b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -378,6 +378,7 @@
assert (-8388608).to_bytes(3, 'little', signed=True) == b'\x00\x00\x80'
raises(OverflowError, (-5).to_bytes, 1, 'big')
raises(ValueError, (-5).to_bytes, 1, 'foo')
+ assert 65535 .to_bytes(length=2, byteorder='big') == b'\xff\xff'
def test_negative_zero(self):
x = eval("-self._long(0)")
diff --git a/rpython/jit/backend/x86/runner.py
b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -44,7 +44,7 @@
if config.translation.jit_profiler == "oprofile":
from rpython.jit.backend.x86 import oprofile
if not oprofile.OPROFILE_AVAILABLE:
- log.WARNING('oprofile support was explicitly enabled, but
oprofile headers seem not to be available')
+ raise Exception('oprofile support was explicitly enabled,
but oprofile headers seem not to be available')
profile_agent = oprofile.OProfileAgent()
self.with_threads = config.translation.thread
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py
b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -11,7 +11,7 @@
from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp
from rpython.jit.metainterp.optimize import InvalidLoop
from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
- AbstractResOp, GuardResOp
+ GuardResOp
from rpython.rlib.objectmodel import we_are_translated
from rpython.jit.metainterp.optimizeopt import info
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py
b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -6,7 +6,7 @@
from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization,
CONST_1,
CONST_0)
from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, AbstractResOp
+from rpython.jit.metainterp.resoperation import rop
from rpython.jit.metainterp.optimizeopt import vstring
from rpython.jit.codewriter.effectinfo import EffectInfo
from rpython.rlib.rarithmetic import intmask
@@ -44,8 +44,9 @@
if b.has_lower and b.has_upper and b.lower == b.upper:
self.make_constant_int(box, b.lower)
- if isinstance(box, AbstractResOp):
- dispatch_bounds_ops(self, box)
+ box1 = self.optimizer.as_operation(box)
+ if box1 is not None:
+ dispatch_bounds_ops(self, box1)
def _optimize_guard_true_false_value(self, op):
return self.emit(op)
@@ -126,10 +127,11 @@
v1, v2 = v2, v1
# if both are constant, the pure optimization will deal with it
if v2.is_constant() and not v1.is_constant():
- if not self.optimizer.is_inputarg(arg1):
+ arg1 = self.optimizer.as_operation(arg1)
+ if arg1 is not None:
if arg1.getopnum() == rop.INT_ADD:
- prod_arg1 = arg1.getarg(0)
- prod_arg2 = arg1.getarg(1)
+ prod_arg1 = self.get_box_replacement(arg1.getarg(0))
+ prod_arg2 = self.get_box_replacement(arg1.getarg(1))
prod_v1 = self.getintbound(prod_arg1)
prod_v2 = self.getintbound(prod_arg2)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py
b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -279,6 +279,7 @@
self.quasi_immutable_deps = None
self.replaces_guard = {}
self._newoperations = []
+ self._emittedoperations = {}
self.optimizer = self
self.optpure = None
self.optheap = None
@@ -292,11 +293,6 @@
self.set_optimizations(optimizations)
self.setup()
- def init_inparg_dict_from(self, lst):
- self.inparg_dict = {}
- for box in lst:
- self.inparg_dict[box] = None
-
def set_optimizations(self, optimizations):
if optimizations:
self.first_optimization = optimizations[0]
@@ -384,9 +380,12 @@
return info.force_box(op, optforce)
return op
- def is_inputarg(self, op):
- return True
- return op in self.inparg_dict
+ def as_operation(self, op):
+ # You should never check "isinstance(op, AbstractResOp" directly.
+ # Instead, use this helper.
+ if isinstance(op, AbstractResOp) and op in self._emittedoperations:
+ return op
+ return None
def get_constant_box(self, box):
box = self.get_box_replacement(box)
@@ -404,6 +403,7 @@
def clear_newoperations(self):
self._newoperations = []
+ self._emittedoperations = {}
def make_equal_to(self, op, newop):
op = self.get_box_replacement(op)
@@ -631,6 +631,7 @@
self._last_guard_op = None
self._really_emitted_operation = op
self._newoperations.append(op)
+ self._emittedoperations[op] = None
def emit_guard_operation(self, op, pendingfields):
guard_op = op # self.replace_op_with(op, op.getopnum())
@@ -675,6 +676,7 @@
return
newop = self.replace_op_with_no_ovf(op)
self._newoperations[-1] = newop
+ self._emittedoperations[newop] = None
def replace_op_with_no_ovf(self, op):
if op.getopnum() == rop.INT_MUL_OVF:
@@ -719,6 +721,7 @@
new_descr = new_op.getdescr()
new_descr.copy_all_attributes_from(old_descr)
self._newoperations[old_op_pos] = new_op
+ self._emittedoperations[new_op] = None
def store_final_boxes_in_guard(self, op, pendingfields):
assert pendingfields is not None
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py
b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -10,7 +10,7 @@
from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL
from rpython.jit.metainterp.optimizeopt.util import _findall,
make_dispatcher_method
from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
- OpHelpers, AbstractResOp
+ OpHelpers
from rpython.rlib.rarithmetic import highest_bit
from rpython.rtyper.lltypesystem import llmemory
from rpython.rtyper import rclass
@@ -389,6 +389,8 @@
def optimize_GUARD_SUBCLASS(self, op):
info = self.getptrinfo(op.getarg(0))
optimizer = self.optimizer
+ # must raise 'InvalidLoop' in all cases where 'info' shows the
+ # class cannot possibly match (see test_issue2926)
if info and info.is_constant():
c = self.get_box_replacement(op.getarg(0))
vtable = optimizer.cpu.ts.cls_of_box(c).getint()
@@ -398,13 +400,29 @@
if info is not None and info.is_about_object():
known_class = info.get_known_class(optimizer.cpu)
if known_class:
+ # Class of 'info' is exactly 'known_class'.
+ # We know statically if the 'guard_subclass' will pass or fail.
if optimizer._check_subclass(known_class.getint(),
op.getarg(1).getint()):
return
+ else:
+ raise InvalidLoop(
+ "GUARD_SUBCLASS(known_class) proven to always fail")
elif info.get_descr() is not None:
- if optimizer._check_subclass(info.get_descr().get_vtable(),
+ # Class of 'info' is either get_descr() or a subclass of it.
+ # We're keeping the 'guard_subclass' at runtime only in the
+ # case where get_descr() is some strict parent class of
+ # the argument to 'guard_subclass'.
+ info_base_descr = info.get_descr().get_vtable()
+ if optimizer._check_subclass(info_base_descr,
op.getarg(1).getint()):
- return
+ return # guard_subclass always passing
+ elif optimizer._check_subclass(op.getarg(1).getint(),
+ info_base_descr):
+ pass # don't know, must keep the 'guard_subclass'
+ else:
+ raise InvalidLoop(
+ "GUARD_SUBCLASS(base_class) proven to always fail")
return self.emit(op)
def optimize_GUARD_NONNULL(self, op):
@@ -490,11 +508,11 @@
def postprocess_GUARD_TRUE(self, op):
box = self.get_box_replacement(op.getarg(0))
- if (isinstance(box, AbstractResOp) and
- box.getopnum() == rop.INT_IS_TRUE):
+ box1 = self.optimizer.as_operation(box)
+ if box1 is not None and box1.getopnum() == rop.INT_IS_TRUE:
# we can't use the (current) range analysis for this because
# "anything but 0" is not a valid range
- self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0)
+ self.pure_from_args(rop.INT_IS_ZERO, [box1.getarg(0)], CONST_0)
self.make_constant(box, CONST_1)
def optimize_GUARD_FALSE(self, op):
@@ -502,11 +520,11 @@
def postprocess_GUARD_FALSE(self, op):
box = self.get_box_replacement(op.getarg(0))
- if (isinstance(box, AbstractResOp) and
- box.getopnum() == rop.INT_IS_ZERO):
+ box1 = self.optimizer.as_operation(box)
+ if box1 is not None and box1.getopnum() == rop.INT_IS_ZERO:
# we can't use the (current) range analysis for this because
# "anything but 0" is not a valid range
- self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1)
+ self.pure_from_args(rop.INT_IS_TRUE, [box1.getarg(0)], CONST_1)
self.make_constant(box, CONST_0)
def optimize_ASSERT_NOT_NONE(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4017,7 +4017,6 @@
strsetitem(p3, i2, i0)
i5 = int_add(i2, 1)
strsetitem(p3, i5, i1)
- ifoo = int_add(i5, 1)
jump(i1, i0, p3)
"""
self.optimize_strunicode_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -74,7 +74,7 @@
print "Loop:"
print '\n'.join([str(o) for o in loop.operations])
print
- if expected_short:
+ if expected_short or getattr(info, 'short_preamble', None):
print "Short Preamble:"
short = info.short_preamble
print '\n'.join([str(o) for o in short])
@@ -1300,7 +1300,7 @@
preamble = """
[i0, p1, p3]
i28 = int_add(i0, 1)
- i29 = int_add(i28, 1)
+ i29 = int_add(i0, 2)
p30 = new_with_vtable(descr=nodesize)
setfield_gc(p30, i28, descr=valuedescr)
setfield_gc(p3, p30, descr=nextdescr)
@@ -1310,7 +1310,7 @@
expected = """
[i0, p1, p3]
i28 = int_add(i0, 1)
- i29 = int_add(i28, 1)
+ i29 = int_add(i0, 2)
p30 = new_with_vtable(descr=nodesize)
setfield_gc(p30, i28, descr=valuedescr)
setfield_gc(p3, p30, descr=nextdescr)
@@ -6392,7 +6392,6 @@
strsetitem(p3, i2, i0)
i5 = int_add(i2, 1)
strsetitem(p3, i5, i1)
- i6 = int_add(i5, 1) # will be killed by the backend
jump(i1, i0, p3)
"""
self.optimize_strunicode_loop(ops, expected, expected)
@@ -9063,6 +9062,7 @@
self.optimize_loop(ops, expected)
def test_same_as_preserves_info_in_the_preamble_2(self):
+ py.test.xfail("less efficient loop, investigate")
ops = """
[i0, p0]
ifoo = getfield_gc_i(p0, descr=valuedescr)
@@ -9499,5 +9499,25 @@
"""
self.optimize_loop(ops, expected)
+ def test_issue2904(self):
+ # we don't store advanced virtualstate information like "i1 = i2 + 1",
+ # which means that the following loop, when unrolled, cannot be
+ # optimized based on the knowledge that "i1 = i2 + 1" from the
+ # preamble---we can't use that knowledge. After the fix, we get
+ # the value "i2 + 1" passed as a third argument, possibly different
+ # from "i1".
+ ops = """
+ [i1, i2]
+ guard_value(i1, 10) []
+ i3 = int_add(i2, 1)
+ jump(i3, i2)
+ """
+ expected = """
+ [i1, i2, i3]
+ guard_value(i1, 10) []
+ jump(i3, i2, i3)
+ """
+ self.optimize_loop(ops, expected)
+
class TestLLtype(OptimizeOptTest, LLtypeMixin):
pass
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py
b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -10,8 +10,7 @@
from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
from rpython.jit.metainterp.optimizeopt.virtualstate import (
VirtualStateConstructor, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp,\
- AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
from rpython.jit.metainterp import compile
from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
have_debug_prints
@@ -22,7 +21,6 @@
if self.optunroll.short_preamble_producer is None:
assert False # unreachable code
op = preamble_op.op
- self.optimizer.inparg_dict[op] = None # XXX ARGH
# special hack for int_add(x, accumulator-const) optimization
self.optunroll.short_preamble_producer.use_box(op,
preamble_op.preamble_op, self)
@@ -144,7 +142,6 @@
except VirtualStatesCantMatch:
raise InvalidLoop("Cannot import state, virtual states don't
match")
self.potential_extra_ops = {}
- self.optimizer.init_inparg_dict_from(label_args)
try:
info, _ = self.optimizer.propagate_all_forward(
trace, call_pure_results, flush=False)
@@ -431,8 +428,9 @@
for box in self._map_args(mapping, short_jump_args)]
def _expand_info(self, arg, infos):
- if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum):
- info = self.optimizer.getinfo(arg.getarg(0))
+ arg1 = self.optimizer.as_operation(arg)
+ if arg1 is not None and rop.is_same_as(arg1.opnum):
+ info = self.optimizer.getinfo(arg1.getarg(0))
else:
info = self.optimizer.getinfo(arg)
if arg in infos:
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py
b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -6,8 +6,7 @@
from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1
from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED
from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
- AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation
from rpython.jit.metainterp.optimizeopt import info
from rpython.rlib.objectmodel import specialize, we_are_translated
from rpython.rlib.unroll import unrolling_iterable
diff --git a/rpython/jit/metainterp/test/test_ajit.py
b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4702,3 +4702,112 @@
res = self.meta_interp(f, [10])
assert res == f(10)
+ def test_cached_info_missing(self):
+ py.test.skip("XXX hitting a non-translated assert in
optimizeopt/heap.py, but seems not to hurt the rest")
+ driver = JitDriver(greens = [],
+ reds=['iterations', 'total', 'c', 'height', 'h'])
+
+ class IntVal:
+ _immutable_fields_ = ['intval']
+ def __init__(self, value):
+ self.intval = value
+
+ def f(height, iterations):
+ height = IntVal(height)
+ c = IntVal(0)
+ h = height
+ total = IntVal(0)
+
+ while True:
+ driver.jit_merge_point(iterations=iterations,
+ total=total, c=c, height=height, h=h)
+ if h.intval > 0:
+ h = IntVal(h.intval - 1)
+ total = IntVal(total.intval + 1)
+ else:
+ c = IntVal(c.intval + 1)
+ if c.intval >= iterations:
+ return total.intval
+ h = height
+
+ res = self.meta_interp(f, [2, 200])
+ assert res == f(2, 200)
+
+ def test_issue2904(self):
+ driver = JitDriver(greens = [],
+ reds=['iterations', 'total', 'c', 'height', 'h'])
+
+ def f(height, iterations):
+ set_param(driver, 'threshold', 4)
+ set_param(driver, 'trace_eagerness', 1)
+ c = 0
+ h = height
+ total = 0
+
+ while True:
+ driver.jit_merge_point(iterations=iterations,
+ total=total, c=c, height=height, h=h)
+ if h != 0:
+ h = h - 1
+ total = total + 1
+ else:
+ c = c + 1
+ if c >= iterations:
+ return total
+ h = height - 1
+
+ res = self.meta_interp(f, [2, 200])
+ assert res == f(2, 200)
+
+ def test_issue2926(self):
+ driver = JitDriver(greens = [], reds=['i', 'total', 'p'])
+
+ class Base(object):
+ def do_stuff(self):
+ return 1000
+ class Int(Base):
+ def __init__(self, intval):
+ self.intval = intval
+ def do_stuff(self):
+ return self.intval
+ class SubInt(Int):
+ pass
+ class Float(Base):
+ def __init__(self, floatval):
+ self.floatval = floatval
+ def do_stuff(self):
+ return int(self.floatval)
+
+ prebuilt = [Int(i) for i in range(10)]
+
+ @dont_look_inside
+ def forget_intbounds(i):
+ return i
+
+ @dont_look_inside
+ def escape(p):
+ pass
+
+ def f(i):
+ total = 0
+ p = Base()
+ while True:
+ driver.jit_merge_point(i=i, total=total, p=p)
+ #print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', i
+ if i == 13:
+ break
+ total += p.do_stuff()
+ j = forget_intbounds(i)
+ if j < 10: # initial loop
+ p = prebuilt[i]
+ p.intval = j
+ elif j < 12:
+ p = Int(i)
+ else:
+ p = Float(3.14)
+ escape(p)
+ i += 1
+ return total
+
+ res = self.meta_interp(f, [0])
+ assert res == f(0)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -62,6 +62,7 @@
# XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w)
import sys
import os
+import time
from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup
from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage
@@ -73,7 +74,6 @@
from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
from rpython.rlib.objectmodel import specialize
from rpython.rlib import rgc
-from rpython.rlib.rtimer import read_timestamp
from rpython.memory.gc.minimarkpage import out_of_memory
#
@@ -192,6 +192,8 @@
# ____________________________________________________________
+
+
class IncrementalMiniMarkGC(MovingGCBase):
_alloc_flavor_ = "raw"
inline_simple_malloc = True
@@ -374,6 +376,7 @@
self.raw_malloc_might_sweep = self.AddressStack()
self.rawmalloced_total_size = r_uint(0)
self.rawmalloced_peak_size = r_uint(0)
+ self.total_gc_time = 0.0
self.gc_state = STATE_SCANNING
#
@@ -1644,7 +1647,7 @@
"""Perform a minor collection: find the objects from the nursery
that remain alive and move them out."""
#
- start = read_timestamp()
+ start = time.time()
debug_start("gc-minor")
#
# All nursery barriers are invalid from this point on. They
@@ -1843,7 +1846,8 @@
self.root_walker.finished_minor_collection()
#
debug_stop("gc-minor")
- duration = read_timestamp() - start
+ duration = time.time() - start
+ self.total_gc_time += duration
self.hooks.fire_gc_minor(
duration=duration,
total_memory_used=total_memory_used,
@@ -2249,7 +2253,7 @@
# Note - minor collections seem fast enough so that one
# is done before every major collection step
def major_collection_step(self, reserving_size=0):
- start = read_timestamp()
+ start = time.time()
debug_start("gc-collect-step")
oldstate = self.gc_state
debug_print("starting gc state: ", GC_STATES[self.gc_state])
@@ -2493,7 +2497,8 @@
debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
debug_stop("gc-collect-step")
- duration = read_timestamp() - start
+ duration = time.time() - start
+ self.total_gc_time += duration
self.hooks.fire_gc_collect_step(
duration=duration,
oldstate=oldstate,
@@ -3000,6 +3005,8 @@
self.ac.total_memory_used))
elif stats_no == rgc.NURSERY_SIZE:
return intmask(self.nursery_size)
+ elif stats_no == rgc.TOTAL_GC_TIME:
+ return int(self.total_gc_time * 1000)
return 0
diff --git a/rpython/memory/gc/test/test_hook.py
b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -70,7 +70,7 @@
assert self.gc.hooks.minors == [
{'total_memory_used': 0, 'pinned_objects': 0}
]
- assert self.gc.hooks.durations[0] > 0
+ assert self.gc.hooks.durations[0] > 0.
self.gc.hooks.reset()
#
# these objects survive, so the total_memory_used is > 0
@@ -103,7 +103,7 @@
]
assert len(self.gc.hooks.durations) == 4 # 4 steps
for d in self.gc.hooks.durations:
- assert d > 0
+ assert d > 0.0
self.gc.hooks.reset()
#
self.stackroots.append(self.malloc(S))
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -1,6 +1,7 @@
import sys
import time
+from rpython.rlib.objectmodel import enforceargs
from rpython.rtyper.extregistry import ExtRegistryEntry
from rpython.rlib.objectmodel import we_are_translated, always_inline
from rpython.rlib.rarithmetic import is_valid_int, r_longlong
@@ -75,6 +76,7 @@
_stop_colors = ""
@always_inline
+@enforceargs(str, bool)
def debug_start(category, timestamp=False):
"""
Start a PYPYLOG section.
@@ -85,6 +87,7 @@
return _debug_start(category, timestamp)
@always_inline
+@enforceargs(str, bool)
def debug_stop(category, timestamp=False):
"""
Stop a PYPYLOG section. See debug_start for docs about timestamp
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -704,7 +704,7 @@
(TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE,
PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY,
TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY,
- NURSERY_SIZE) = range(10)
+ NURSERY_SIZE, TOTAL_GC_TIME) = range(11)
@not_rpython
def get_stats(stat_no):
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -708,7 +708,7 @@
errno = _c.geterrno()
timeout = self.timeout
if (timeout > 0.0 and res < 0 and
- errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)):
+ errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)):
tv = rffi.make(_c.timeval)
rffi.setintfield(tv, 'c_tv_sec', int(timeout))
rffi.setintfield(tv, 'c_tv_usec',
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -136,7 +136,10 @@
void = lltype.nullptr(rffi.VOIDP.TO)
result = -1.0
if HAVE_GETTIMEOFDAY:
- with lltype.scoped_alloc(TIMEVAL) as t:
+ # NB: can't use lltype.scoped_malloc, because that will allocate the
+ # with handler in the GC, but we want to use time.time from gc.collect!
+ t = lltype.malloc(TIMEVAL, flavor='raw')
+ try:
errcode = -1
if GETTIMEOFDAY_NO_TZ:
errcode = c_gettimeofday(t)
@@ -145,13 +148,18 @@
if rffi.cast(rffi.LONG, errcode) == 0:
result = decode_timeval(t)
+ finally:
+ lltype.free(t, flavor='raw')
if result != -1:
return result
else: # assume using ftime(3)
- with lltype.scoped_alloc(TIMEB) as t:
+ t = lltype.malloc(TIMEB, flavor='raw')
+ try:
c_ftime(t)
result = (float(intmask(t.c_time)) +
float(intmask(t.c_millitm)) * 0.001)
+ finally:
+ lltype.free(t, flavor='raw')
return result
return float(c_time(void))
diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py
b/rpython/translator/backendopt/test/test_mallocprediction.py
--- a/rpython/translator/backendopt/test/test_mallocprediction.py
+++ b/rpython/translator/backendopt/test/test_mallocprediction.py
@@ -179,7 +179,7 @@
t, graph = rtype(entry_point, [int])
total0 = preparation(t, t.graphs)
total = clever_inlining_and_malloc_removal(t)
- assert total0 + total == 10
+ assert total0 + total == 9
def test_loop():
l = [10, 12, 15, 1]
diff --git a/rpython/translator/c/test/test_newgc.py
b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1812,7 +1812,20 @@
res = self.run("ignore_finalizer")
assert res == 1 # translated: x1 is removed from the list
+ def define_total_gc_time(cls):
+ def f():
+ l = []
+ for i in range(1000000):
+ l.append(str(i))
+ l = []
+ for i in range(10):
+ rgc.collect()
+ return rgc.get_stats(rgc.TOTAL_GC_TIME)
+ return f
+ def test_total_gc_time(self):
+ res = self.run("total_gc_time")
+ assert res > 0 # should take a few microseconds
# ____________________________________________________________________
class TaggedPointersTest(object):
diff --git a/rpython/translator/c/test/test_standalone.py
b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -521,11 +521,9 @@
assert path.check(file=0)
def test_debug_start_stop_timestamp(self):
- import sys
- import time
from rpython.rlib.rtimer import read_timestamp
def entry_point(argv):
- timestamp = int(argv[1])
+ timestamp = bool(int(argv[1]))
ts1 = debug_start("foo", timestamp=timestamp)
ts2 = read_timestamp()
ts3 = debug_stop("foo", timestamp=timestamp)
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
