Author: Ronan Lamy <[email protected]> Branch: py3.5 Changeset: r93839:dd3b9bfab6a0 Date: 2018-02-20 02:47 +0000 http://bitbucket.org/pypy/pypy/changeset/dd3b9bfab6a0/
Log: hg merge default diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py rename from pypy/module/test_lib_pypy/pyrepl/test_functional.py rename to extra_tests/test_pyrepl/test_functional.py --- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py +++ b/extra_tests/test_pyrepl/test_functional.py @@ -7,7 +7,8 @@ import sys -def pytest_funcarg__child(request): [email protected]() +def child(): try: import pexpect except ImportError: diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,137 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -2,9 +2,10 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr, - decref) + decref, as_pyobj, incref) from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct, - PyObjectFields, cts, parse_dir, bootstrap_function, slot_function) + PyObjectFields, cts, parse_dir, bootstrap_function, slot_function, + Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError @@ -31,6 +32,10 @@ w_type = space.getattr(w_datetime, space.newtext("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + # convenient place to modify this, needed since the make_typedescr attach + # links the "wrong" struct to W_DateTime_Date, which in turn is needed + # because datetime, with a tzinfo entry, inherits from date, without one + datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO) w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( @@ -128,6 +133,7 @@ # W_DateTime_Date->tp_dealloc make_typedescr(W_DateTime_Date.typedef, basestruct=PyDateTime_DateTime.TO, + attach=type_attach, dealloc=date_dealloc, ) @@ -138,8 +144,10 @@ def type_attach(space, py_obj, w_obj, w_userdata=None): '''Fills a newly allocated py_obj from the w_obj - Can be called with a datetime, or a time ''' + if space.type(w_obj).name == 'date': + # No tzinfo + return py_datetime = rffi.cast(PyDateTime_Time, py_obj) w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) if space.is_none(w_tzinfo): diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -82,16 +82,6 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) - def test_tzinfo(self, space): - w_tzinfo = space.appexec( - [], """(): - from datetime import tzinfo - return tzinfo() - """) - assert PyTZInfo_Check(space, w_tzinfo) - assert PyTZInfo_CheckExact(space, w_tzinfo) - assert not PyTZInfo_Check(space, space.w_None) - class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -272,3 +262,81 @@ 2000, 6, 6, 6, 6, 6, 6) assert module.test_time_macros() == datetime.time(6, 6, 6, 6) assert module.test_delta_macros() == datetime.timedelta(6, 6, 6) + + def test_tzinfo(self): + module = self.import_extension('foo', [ + ("time_with_tzinfo", "METH_O", + """ PyDateTime_IMPORT; + return PyDateTimeAPI->Time_FromTime( + 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); + """), + ("datetime_with_tzinfo", "METH_O", + """ + PyObject * obj; + int tzrefcnt = args->ob_refcnt; + PyDateTime_IMPORT; + obj = PyDateTimeAPI->DateTime_FromDateAndTime( + 2000, 6, 6, 6, 6, 6, 6, args, + PyDateTimeAPI->DateTimeType); + if (!((PyDateTime_DateTime*)obj)->hastzinfo) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "missing tzinfo"); + return NULL; + } + if (((PyDateTime_DateTime*)obj)->tzinfo->ob_refcnt <= tzrefcnt) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "tzinfo refcnt not incremented"); + return NULL; + } + return obj; + + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time + # copied from datetime documentation + class GMT1(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=1) + self.dst(dt) + def dst(self, dt): + return timedelta(0) + def tzname(self,dt): + return "GMT +1" + gmt1 = GMT1() + dt1 = module.time_with_tzinfo(gmt1) + assert dt1 == time(6, 6, 6, 6, gmt1) + assert '+01' in str(dt1) + assert module.datetime_with_tzinfo(gmt1) == datetime( + 2000, 6, 6, 6, 6, 6, 6, gmt1) + + def test_checks(self): + module = self.import_extension('foo', [ + ("checks", "METH_O", + """ PyDateTime_IMPORT; + return PyTuple_Pack(10, + PyBool_FromLong(PyDateTime_Check(args)), + PyBool_FromLong(PyDateTime_CheckExact(args)), + PyBool_FromLong(PyDate_Check(args)), + PyBool_FromLong(PyDate_CheckExact(args)), + PyBool_FromLong(PyTime_Check(args)), + PyBool_FromLong(PyTime_CheckExact(args)), + PyBool_FromLong(PyDelta_Check(args)), + PyBool_FromLong(PyDelta_CheckExact(args)), + PyBool_FromLong(PyTZInfo_Check(args)), + PyBool_FromLong(PyTZInfo_CheckExact(args)) + ); + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time, date + o = date(1, 1, 1) + assert module.checks(o) == (False,) * 2 + (True,) * 2 + (False,) * 6 + o = time(1, 1, 1) + assert module.checks(o) == (False,) * 4 + (True,) * 2 + (False,) * 4 + o = timedelta(1, 1, 1) + assert module.checks(o) == (False,) * 6 + (True,) * 2 + (False,) * 2 + o = datetime(1, 1, 1) + assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) + o = tzinfo() + assert module.checks(o) == (False,) * 8 + (True,) * 2 + diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -72,7 +72,7 @@ def __repr__(self): if self._s.total_memory_pressure != -1: - extra = "\nmemory pressure: %s" % self.total_memory_pressure + extra = "\n memory pressure: %s" % self.total_memory_pressure else: extra = "" return """Total memory consumed: @@ -109,5 +109,5 @@ self.memory_allocated_sum) -def get_stats(): - return GcStats(gc._get_stats()) +def get_stats(memory_pressure=False): + return GcStats(gc._get_stats(memory_pressure=memory_pressure)) _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
