Author: Matti Picus <matti.pi...@gmail.com> Branch: py3.5 Changeset: r95446:1300ca1763e1 Date: 2018-12-09 11:13 +0200 http://bitbucket.org/pypy/pypy/changeset/1300ca1763e1/
Log: merge default into branch diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy", allow_module_level=True) + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -51,3 +51,8 @@ .. branch: rlock-in-rpython Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -386,7 +386,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -603,8 +603,14 @@ warnoptions = pythonwarnings.split(',') + warnoptions if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + +@pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs +@pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in str(e.value) + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in str(excinfo.value) - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in str(e.value) +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in str(excinfo.value) - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + +@pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) + with pytest.raises(ValueError) as e: + con.execute('insert into foo(x) values (?)', 2) + assert str(e.value) == 'parameters are of unsupported type' + +def test_explicit_begin(con): + con.execute('BEGIN') + con.execute('BEGIN ') + con.execute('BEGIN') + con.commit() + con.execute('BEGIN') + con.commit() + +def test_row_factory_use(con): + con.row_factory = 42 + con.execute('select 1') + +def test_returning_blob_must_own_memory(con): + import gc + con.create_function("returnblob", 0, lambda: buffer("blob")) + cur = con.execute("select returnblob()") + val = cur.fetchone()[0] + for i in range(5): + gc.collect() + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'l', 'o', 'b') + # in theory 'val' should be a read-write buffer + # but it's not right now + if not hasattr(_sqlite3, '_ffi'): + val[1] = 'X' + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'X', 'o', 'b') + +def test_description_after_fetchall(con): + cur = con.cursor() + assert cur.description is None + cur.execute("select 42").fetchall() + assert cur.description is not None + +def test_executemany_lastrowid(con): + cur = con.cursor() + cur.execute("create table test(a)") + cur.executemany("insert into test values (?)", [[1], [2], [3]]) + assert cur.lastrowid is None + # issue 2682 + cur.execute('''insert + into test + values (?) + ''', (1, )) + assert cur.lastrowid is not None + cur.execute('''insert\t into test values (?) ''', (1, )) + assert cur.lastrowid is not None + +def test_authorizer_bad_value(con): + def authorizer_cb(action, arg1, arg2, dbname, source): + return 42 + con.set_authorizer(authorizer_cb) + with pytest.raises(_sqlite3.OperationalError) as e: + con.execute('select 123') + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) >= (3, 6, 14): + assert str(e.value) == 'authorizer malfunction' + else: + assert str(e.value) == \ + ("illegal return value (1) from the authorization function - " + "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") + +def test_issue1573(con): + cur = con.cursor() + cur.execute(u'SELECT 1 as méil') + assert cur.description[0][0] == u"méil".encode('utf-8') + +def test_adapter_exception(con): + def cast(obj): + raise ZeroDivisionError + + _sqlite3.register_adapter(int, cast) + try: cur = con.cursor() - with pytest.raises(StopIteration): - next(cur) + cur.execute("select ?", (4,)) + val = cur.fetchone()[0] + # Adapter error is ignored, and parameter is passed as is. + assert val == 4 + assert type(val) is int + finally: + del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - cur.execute('select 1') - next(cur) - with pytest.raises(StopIteration): - next(cur) +def test_null_character(con): + if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): + pytest.skip("_sqlite3 too old") + with raises(ValueError) as excinfo: + con("\0select 1") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + con("select 1\0") + assert str(excinfo.value) == "the query contains a null character" + cur = con.cursor() + with raises(ValueError) as excinfo: + cur.execute("\0select 2") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + cur.execute("select 2\0") + assert str(excinfo.value) == "the query contains a null character" - cur.execute('select 1') - con.commit() - next(cur) - with pytest.raises(StopIteration): - next(cur) - - with pytest.raises(_sqlite3.ProgrammingError): - cur.executemany('select 1', []) - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('create table test(ing)') - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('insert into test values(1)') - con.commit() - with pytest.raises(StopIteration): - next(cur) - - def test_cursor_after_close(self, con): - cur = con.execute('select 1') - cur.close() - con.close() - pytest.raises(_sqlite3.ProgrammingError, "cur.close()") - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)") - pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)") - - @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')") - def test_connection_del(self, tmpdir): - """For issue1325.""" - import os - import gc - try: - import resource - except ImportError: - pytest.skip("needs resource module") - - limit = resource.getrlimit(resource.RLIMIT_NOFILE) - try: - fds = 0 - while True: - fds += 1 - resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) - try: - for p in os.pipe(): os.close(p) - except OSError: - assert fds < 100 - else: - break - - def open_many(cleanup): - con = [] - for i in range(3): - con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) - if cleanup: - con[i] = None - gc.collect(); gc.collect() - - pytest.raises(_sqlite3.OperationalError, open_many, False) - gc.collect(); gc.collect() - open_many(True) - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, limit) - - def test_on_conflict_rollback_executemany(self, con): - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) < (3, 2, 2): - pytest.skip("requires sqlite3 version >= 3.2.2") - con.execute("create table foo(x, unique(x) on conflict rollback)") - con.execute("insert into foo(x) values (1)") - try: - con.executemany("insert into foo(x) values (?)", [[1]]) - except _sqlite3.DatabaseError: - pass - con.execute("insert into foo(x) values (2)") - try: - con.commit() - except _sqlite3.OperationalError: - pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") - - def test_statement_arg_checking(self, con): - with pytest.raises(_sqlite3.Warning) as e: - con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' - with pytest.raises(ValueError) as e: - con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' - - def test_statement_param_checking(self, con): - con.execute('create table foo(x)') - con.execute('insert into foo(x) values (?)', [2]) - con.execute('insert into foo(x) values (?)', (2,)) - class seq(object): - def __len__(self): - return 1 - def __getitem__(self, key): - return 2 - con.execute('insert into foo(x) values (?)', seq()) - del seq.__len__ - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', seq()) - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', {2:2}) - with pytest.raises(ValueError) as e: - con.execute('insert into foo(x) values (?)', 2) - assert str(e.value) == 'parameters are of unsupported type' - - def test_explicit_begin(self, con): - con.execute('BEGIN') - con.execute('BEGIN ') - con.execute('BEGIN') - con.commit() - con.execute('BEGIN') - con.commit() - - def test_row_factory_use(self, con): - con.row_factory = 42 - con.execute('select 1') - - def test_returning_blob_must_own_memory(self, con): - import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) - cur = con.execute("select returnblob()") - val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') - - def test_description_after_fetchall(self, con): - cur = con.cursor() - assert cur.description is None - cur.execute("select 42").fetchall() - assert cur.description is not None - - def test_executemany_lastrowid(self, con): - cur = con.cursor() - cur.execute("create table test(a)") - cur.executemany("insert into test values (?)", [[1], [2], [3]]) - assert cur.lastrowid is None - # issue 2682 - cur.execute('''insert - into test - values (?) - ''', (1, )) - assert cur.lastrowid is not None - cur.execute('''insert\t into test values (?) ''', (1, )) - assert cur.lastrowid is not None - - def test_authorizer_bad_value(self, con): - def authorizer_cb(action, arg1, arg2, dbname, source): - return 42 - con.set_authorizer(authorizer_cb) - with pytest.raises(_sqlite3.OperationalError) as e: - con.execute('select 123') - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) >= (3, 6, 14): - assert str(e.value) == 'authorizer malfunction' - else: - assert str(e.value) == \ - ("illegal return value (1) from the authorization function - " - "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") - - def test_issue1573(self, con): - cur = con.cursor() - cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') - - def test_adapter_exception(self, con): - def cast(obj): - raise ZeroDivisionError - - _sqlite3.register_adapter(int, cast) - try: - cur = con.cursor() - cur.execute("select ?", (4,)) - val = cur.fetchone()[0] - # Adapter error is ignored, and parameter is passed as is. - assert val == 4 - assert type(val) is int - finally: - del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - - def test_null_character(self, con): - if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): - pytest.skip("_sqlite3 too old") - exc = raises(ValueError, con, "\0select 1") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, con, "select 1\0") - assert str(exc.value) == "the query contains a null character" - cur = con.cursor() - exc = raises(ValueError, cur.execute, "\0select 2") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, cur.execute, "select 2\0") - assert str(exc.value) == "the query contains a null character" - - def test_close_in_del_ordering(self): - import gc - class SQLiteBackend(object): - success = False - def __init__(self): - self.connection = _sqlite3.connect(":memory:") - def close(self): - self.connection.close() - def __del__(self): - self.close() - SQLiteBackend.success = True - def create_db_if_needed(self): - conn = self.connection - cursor = conn.cursor() - cursor.execute(""" - create table if not exists nameoftable(value text) - """) - cursor.close() - conn.commit() - SQLiteBackend().create_db_if_needed() - gc.collect() - gc.collect() - assert SQLiteBackend.success - - -class TestSQLiteHost(BaseTestSQLite): - def setup_class(cls): - global _sqlite3 - import _sqlite3 - - -class TestSQLitePyPy(BaseTestSQLite): - def setup_class(cls): - if sys.version_info < (2, 7): - pytest.skip("_sqlite3 requires Python 2.7") - - try: - from lib_pypy import _sqlite3_cffi - except ImportError: - # On CPython, "pip install cffi". On old PyPy's, no chance - pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py " - "manually first") - - global _sqlite3 - from lib_pypy import _sqlite3 +def test_close_in_del_ordering(): + import gc + class SQLiteBackend(object): + success = False + def __init__(self): + self.connection = _sqlite3.connect(":memory:") + def close(self): + self.connection.close() + def __del__(self): + self.close() + SQLiteBackend.success = True + def create_db_if_needed(self): + conn = self.connection + cursor = conn.cursor() + cursor.execute(""" + create table if not exists nameoftable(value text) + """) + cursor.close() + conn.commit() + SQLiteBackend().create_db_if_needed() + gc.collect() + gc.collect() + assert SQLiteBackend.success diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -44,7 +44,7 @@ if config.translation.jit_profiler == "oprofile": from rpython.jit.backend.x86 import oprofile if not oprofile.OPROFILE_AVAILABLE: - log.WARNING('oprofile support was explicitly enabled, but oprofile headers seem not to be available') + raise Exception('oprofile support was explicitly enabled, but oprofile headers seem not to be available') profile_agent = oprofile.OProfileAgent() self.with_threads = config.translation.thread diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -11,7 +11,7 @@ from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import InvalidLoop from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\ - AbstractResOp, GuardResOp + GuardResOp from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.optimizeopt import info diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -6,7 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1, CONST_0) from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, AbstractResOp +from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.rlib.rarithmetic import intmask @@ -44,8 +44,9 @@ if b.has_lower and b.has_upper and b.lower == b.upper: self.make_constant_int(box, b.lower) - if isinstance(box, AbstractResOp): - dispatch_bounds_ops(self, box) + box1 = self.optimizer.as_operation(box) + if box1 is not None: + dispatch_bounds_ops(self, box1) def _optimize_guard_true_false_value(self, op): return self.emit(op) @@ -126,10 +127,11 @@ v1, v2 = v2, v1 # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): - if not self.optimizer.is_inputarg(arg1): + arg1 = self.optimizer.as_operation(arg1) + if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: - prod_arg1 = arg1.getarg(0) - prod_arg2 = arg1.getarg(1) + prod_arg1 = self.get_box_replacement(arg1.getarg(0)) + prod_arg2 = self.get_box_replacement(arg1.getarg(1)) prod_v1 = self.getintbound(prod_arg1) prod_v2 = self.getintbound(prod_arg2) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -279,6 +279,7 @@ self.quasi_immutable_deps = None self.replaces_guard = {} self._newoperations = [] + self._emittedoperations = {} self.optimizer = self self.optpure = None self.optheap = None @@ -292,11 +293,6 @@ self.set_optimizations(optimizations) self.setup() - def init_inparg_dict_from(self, lst): - self.inparg_dict = {} - for box in lst: - self.inparg_dict[box] = None - def set_optimizations(self, optimizations): if optimizations: self.first_optimization = optimizations[0] @@ -384,9 +380,12 @@ return info.force_box(op, optforce) return op - def is_inputarg(self, op): - return True - return op in self.inparg_dict + def as_operation(self, op): + # You should never check "isinstance(op, AbstractResOp" directly. + # Instead, use this helper. + if isinstance(op, AbstractResOp) and op in self._emittedoperations: + return op + return None def get_constant_box(self, box): box = self.get_box_replacement(box) @@ -404,6 +403,7 @@ def clear_newoperations(self): self._newoperations = [] + self._emittedoperations = {} def make_equal_to(self, op, newop): op = self.get_box_replacement(op) @@ -631,6 +631,7 @@ self._last_guard_op = None self._really_emitted_operation = op self._newoperations.append(op) + self._emittedoperations[op] = None def emit_guard_operation(self, op, pendingfields): guard_op = op # self.replace_op_with(op, op.getopnum()) @@ -675,6 +676,7 @@ return newop = self.replace_op_with_no_ovf(op) self._newoperations[-1] = newop + self._emittedoperations[newop] = None def replace_op_with_no_ovf(self, op): if op.getopnum() == rop.INT_MUL_OVF: @@ -719,6 +721,7 @@ new_descr = new_op.getdescr() new_descr.copy_all_attributes_from(old_descr) self._newoperations[old_op_pos] = new_op + self._emittedoperations[new_op] = None def store_final_boxes_in_guard(self, op, pendingfields): assert pendingfields is not None diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -10,7 +10,7 @@ from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ - OpHelpers, AbstractResOp + OpHelpers from rpython.rlib.rarithmetic import highest_bit from rpython.rtyper.lltypesystem import llmemory from rpython.rtyper import rclass @@ -389,6 +389,8 @@ def optimize_GUARD_SUBCLASS(self, op): info = self.getptrinfo(op.getarg(0)) optimizer = self.optimizer + # must raise 'InvalidLoop' in all cases where 'info' shows the + # class cannot possibly match (see test_issue2926) if info and info.is_constant(): c = self.get_box_replacement(op.getarg(0)) vtable = optimizer.cpu.ts.cls_of_box(c).getint() @@ -398,13 +400,29 @@ if info is not None and info.is_about_object(): known_class = info.get_known_class(optimizer.cpu) if known_class: + # Class of 'info' is exactly 'known_class'. + # We know statically if the 'guard_subclass' will pass or fail. if optimizer._check_subclass(known_class.getint(), op.getarg(1).getint()): return + else: + raise InvalidLoop( + "GUARD_SUBCLASS(known_class) proven to always fail") elif info.get_descr() is not None: - if optimizer._check_subclass(info.get_descr().get_vtable(), + # Class of 'info' is either get_descr() or a subclass of it. + # We're keeping the 'guard_subclass' at runtime only in the + # case where get_descr() is some strict parent class of + # the argument to 'guard_subclass'. + info_base_descr = info.get_descr().get_vtable() + if optimizer._check_subclass(info_base_descr, op.getarg(1).getint()): - return + return # guard_subclass always passing + elif optimizer._check_subclass(op.getarg(1).getint(), + info_base_descr): + pass # don't know, must keep the 'guard_subclass' + else: + raise InvalidLoop( + "GUARD_SUBCLASS(base_class) proven to always fail") return self.emit(op) def optimize_GUARD_NONNULL(self, op): @@ -490,11 +508,11 @@ def postprocess_GUARD_TRUE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_TRUE): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_TRUE: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0) + self.pure_from_args(rop.INT_IS_ZERO, [box1.getarg(0)], CONST_0) self.make_constant(box, CONST_1) def optimize_GUARD_FALSE(self, op): @@ -502,11 +520,11 @@ def postprocess_GUARD_FALSE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_ZERO): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_ZERO: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1) + self.pure_from_args(rop.INT_IS_TRUE, [box1.getarg(0)], CONST_1) self.make_constant(box, CONST_0) def optimize_ASSERT_NOT_NONE(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4017,7 +4017,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - ifoo = int_add(i5, 1) jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -74,7 +74,7 @@ print "Loop:" print '\n'.join([str(o) for o in loop.operations]) print - if expected_short: + if expected_short or getattr(info, 'short_preamble', None): print "Short Preamble:" short = info.short_preamble print '\n'.join([str(o) for o in short]) @@ -1300,7 +1300,7 @@ preamble = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -1310,7 +1310,7 @@ expected = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -6392,7 +6392,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - i6 = int_add(i5, 1) # will be killed by the backend jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected, expected) @@ -9063,6 +9062,7 @@ self.optimize_loop(ops, expected) def test_same_as_preserves_info_in_the_preamble_2(self): + py.test.xfail("less efficient loop, investigate") ops = """ [i0, p0] ifoo = getfield_gc_i(p0, descr=valuedescr) @@ -9499,5 +9499,25 @@ """ self.optimize_loop(ops, expected) + def test_issue2904(self): + # we don't store advanced virtualstate information like "i1 = i2 + 1", + # which means that the following loop, when unrolled, cannot be + # optimized based on the knowledge that "i1 = i2 + 1" from the + # preamble---we can't use that knowledge. After the fix, we get + # the value "i2 + 1" passed as a third argument, possibly different + # from "i1". + ops = """ + [i1, i2] + guard_value(i1, 10) [] + i3 = int_add(i2, 1) + jump(i3, i2) + """ + expected = """ + [i1, i2, i3] + guard_value(i1, 10) [] + jump(i3, i2, i3) + """ + self.optimize_loop(ops, expected) + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -10,8 +10,7 @@ from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo from rpython.jit.metainterp.optimizeopt.virtualstate import ( VirtualStateConstructor, VirtualStatesCantMatch) -from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp from rpython.jit.metainterp import compile from rpython.rlib.debug import debug_print, debug_start, debug_stop,\ have_debug_prints @@ -22,7 +21,6 @@ if self.optunroll.short_preamble_producer is None: assert False # unreachable code op = preamble_op.op - self.optimizer.inparg_dict[op] = None # XXX ARGH # special hack for int_add(x, accumulator-const) optimization self.optunroll.short_preamble_producer.use_box(op, preamble_op.preamble_op, self) @@ -144,7 +142,6 @@ except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} - self.optimizer.init_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( trace, call_pure_results, flush=False) @@ -431,8 +428,9 @@ for box in self._map_args(mapping, short_jump_args)] def _expand_info(self, arg, infos): - if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum): - info = self.optimizer.getinfo(arg.getarg(0)) + arg1 = self.optimizer.as_operation(arg) + if arg1 is not None and rop.is_same_as(arg1.opnum): + info = self.optimizer.getinfo(arg1.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -6,8 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, ResOperation,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.optimizeopt import info from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.unroll import unrolling_iterable diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4702,3 +4702,112 @@ res = self.meta_interp(f, [10]) assert res == f(10) + def test_cached_info_missing(self): + py.test.skip("XXX hitting a non-translated assert in optimizeopt/heap.py, but seems not to hurt the rest") + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + class IntVal: + _immutable_fields_ = ['intval'] + def __init__(self, value): + self.intval = value + + def f(height, iterations): + height = IntVal(height) + c = IntVal(0) + h = height + total = IntVal(0) + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h.intval > 0: + h = IntVal(h.intval - 1) + total = IntVal(total.intval + 1) + else: + c = IntVal(c.intval + 1) + if c.intval >= iterations: + return total.intval + h = height + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2904(self): + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + def f(height, iterations): + set_param(driver, 'threshold', 4) + set_param(driver, 'trace_eagerness', 1) + c = 0 + h = height + total = 0 + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h != 0: + h = h - 1 + total = total + 1 + else: + c = c + 1 + if c >= iterations: + return total + h = height - 1 + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2926(self): + driver = JitDriver(greens = [], reds=['i', 'total', 'p']) + + class Base(object): + def do_stuff(self): + return 1000 + class Int(Base): + def __init__(self, intval): + self.intval = intval + def do_stuff(self): + return self.intval + class SubInt(Int): + pass + class Float(Base): + def __init__(self, floatval): + self.floatval = floatval + def do_stuff(self): + return int(self.floatval) + + prebuilt = [Int(i) for i in range(10)] + + @dont_look_inside + def forget_intbounds(i): + return i + + @dont_look_inside + def escape(p): + pass + + def f(i): + total = 0 + p = Base() + while True: + driver.jit_merge_point(i=i, total=total, p=p) + #print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', i + if i == 13: + break + total += p.do_stuff() + j = forget_intbounds(i) + if j < 10: # initial loop + p = prebuilt[i] + p.intval = j + elif j < 12: + p = Int(i) + else: + p = Float(3.14) + escape(p) + i += 1 + return total + + res = self.meta_interp(f, [0]) + assert res == f(0) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,6 +62,7 @@ # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys import os +import time from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -73,7 +74,6 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc -from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -192,6 +192,8 @@ # ____________________________________________________________ + + class IncrementalMiniMarkGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -374,6 +376,7 @@ self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) self.rawmalloced_peak_size = r_uint(0) + self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING # @@ -1644,7 +1647,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # - start = read_timestamp() + start = time.time() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1843,7 +1846,8 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_minor( duration=duration, total_memory_used=total_memory_used, @@ -2249,7 +2253,7 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): - start = read_timestamp() + start = time.time() debug_start("gc-collect-step") oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) @@ -2493,7 +2497,8 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_collect_step( duration=duration, oldstate=oldstate, @@ -3000,6 +3005,8 @@ self.ac.total_memory_used)) elif stats_no == rgc.NURSERY_SIZE: return intmask(self.nursery_size) + elif stats_no == rgc.TOTAL_GC_TIME: + return int(self.total_gc_time * 1000) return 0 diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -70,7 +70,7 @@ assert self.gc.hooks.minors == [ {'total_memory_used': 0, 'pinned_objects': 0} ] - assert self.gc.hooks.durations[0] > 0 + assert self.gc.hooks.durations[0] > 0. self.gc.hooks.reset() # # these objects survive, so the total_memory_used is > 0 @@ -103,7 +103,7 @@ ] assert len(self.gc.hooks.durations) == 4 # 4 steps for d in self.gc.hooks.durations: - assert d > 0 + assert d > 0.0 self.gc.hooks.reset() # self.stackroots.append(self.malloc(S)) diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,6 +1,7 @@ import sys import time +from rpython.rlib.objectmodel import enforceargs from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rlib.objectmodel import we_are_translated, always_inline from rpython.rlib.rarithmetic import is_valid_int, r_longlong @@ -75,6 +76,7 @@ _stop_colors = "" @always_inline +@enforceargs(str, bool) def debug_start(category, timestamp=False): """ Start a PYPYLOG section. @@ -85,6 +87,7 @@ return _debug_start(category, timestamp) @always_inline +@enforceargs(str, bool) def debug_stop(category, timestamp=False): """ Stop a PYPYLOG section. See debug_start for docs about timestamp diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -704,7 +704,7 @@ (TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY, TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY, - NURSERY_SIZE) = range(10) + NURSERY_SIZE, TOTAL_GC_TIME) = range(11) @not_rpython def get_stats(stat_no): diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -136,7 +136,10 @@ void = lltype.nullptr(rffi.VOIDP.TO) result = -1.0 if HAVE_GETTIMEOFDAY: - with lltype.scoped_alloc(TIMEVAL) as t: + # NB: can't use lltype.scoped_malloc, because that will allocate the + # with handler in the GC, but we want to use time.time from gc.collect! + t = lltype.malloc(TIMEVAL, flavor='raw') + try: errcode = -1 if GETTIMEOFDAY_NO_TZ: errcode = c_gettimeofday(t) @@ -145,13 +148,18 @@ if rffi.cast(rffi.LONG, errcode) == 0: result = decode_timeval(t) + finally: + lltype.free(t, flavor='raw') if result != -1: return result else: # assume using ftime(3) - with lltype.scoped_alloc(TIMEB) as t: + t = lltype.malloc(TIMEB, flavor='raw') + try: c_ftime(t) result = (float(intmask(t.c_time)) + float(intmask(t.c_millitm)) * 0.001) + finally: + lltype.free(t, flavor='raw') return result return float(c_time(void)) diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py b/rpython/translator/backendopt/test/test_mallocprediction.py --- a/rpython/translator/backendopt/test/test_mallocprediction.py +++ b/rpython/translator/backendopt/test/test_mallocprediction.py @@ -179,7 +179,7 @@ t, graph = rtype(entry_point, [int]) total0 = preparation(t, t.graphs) total = clever_inlining_and_malloc_removal(t) - assert total0 + total == 10 + assert total0 + total == 9 def test_loop(): l = [10, 12, 15, 1] diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,7 +1812,20 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_total_gc_time(cls): + def f(): + l = [] + for i in range(1000000): + l.append(str(i)) + l = [] + for i in range(10): + rgc.collect() + return rgc.get_stats(rgc.TOTAL_GC_TIME) + return f + def test_total_gc_time(self): + res = self.run("total_gc_time") + assert res > 0 # should take a few microseconds # ____________________________________________________________________ class TaggedPointersTest(object): diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -521,11 +521,9 @@ assert path.check(file=0) def test_debug_start_stop_timestamp(self): - import sys - import time from rpython.rlib.rtimer import read_timestamp def entry_point(argv): - timestamp = int(argv[1]) + timestamp = bool(int(argv[1])) ts1 = debug_start("foo", timestamp=timestamp) ts2 = read_timestamp() ts3 = debug_stop("foo", timestamp=timestamp) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit