Author: Ronan Lamy <ronan.l...@gmail.com>
Branch: py3.5
Changeset: r93839:dd3b9bfab6a0
Date: 2018-02-20 02:47 +0000
http://bitbucket.org/pypy/pypy/changeset/dd3b9bfab6a0/

Log:    hg merge default

diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py 
b/extra_tests/test_pyrepl/__init__.py
rename from pypy/module/test_lib_pypy/pyrepl/__init__.py
rename to extra_tests/test_pyrepl/__init__.py
--- a/pypy/module/test_lib_pypy/pyrepl/__init__.py
+++ b/extra_tests/test_pyrepl/__init__.py
@@ -1,3 +1,1 @@
-import sys
-import lib_pypy.pyrepl
-sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl']
+
diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py 
b/extra_tests/test_pyrepl/infrastructure.py
rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py
rename to extra_tests/test_pyrepl/infrastructure.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py 
b/extra_tests/test_pyrepl/test_basic.py
rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py
rename to extra_tests/test_pyrepl/test_basic.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py 
b/extra_tests/test_pyrepl/test_bugs.py
rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py
rename to extra_tests/test_pyrepl/test_bugs.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py 
b/extra_tests/test_pyrepl/test_functional.py
rename from pypy/module/test_lib_pypy/pyrepl/test_functional.py
rename to extra_tests/test_pyrepl/test_functional.py
--- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py
+++ b/extra_tests/test_pyrepl/test_functional.py
@@ -7,7 +7,8 @@
 import sys
 
 
-def pytest_funcarg__child(request):
+@pytest.fixture()
+def child():
     try:
         import pexpect
     except ImportError:
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py 
b/extra_tests/test_pyrepl/test_keymap.py
rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py
rename to extra_tests/test_pyrepl/test_keymap.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py 
b/extra_tests/test_pyrepl/test_reader.py
rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py
rename to extra_tests/test_pyrepl/test_reader.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py 
b/extra_tests/test_pyrepl/test_readline.py
rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py
rename to extra_tests/test_pyrepl/test_readline.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py 
b/extra_tests/test_pyrepl/test_wishes.py
rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py
rename to extra_tests/test_pyrepl/test_wishes.py
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -1,17 +1,137 @@
-Garbage collector configuration
-===============================
+Garbage collector documentation and configuration
+=================================================
+
+
+Incminimark
+-----------
+
+PyPy's default garbage collector is called incminimark - it's an incremental,
+generational moving collector. Here we hope to explain a bit how it works
+and how it can be tuned to suit the workload.
+
+Incminimark first allocates objects in so called *nursery* - place for young
+objects, where allocation is very cheap, being just a pointer bump. The nursery
+size is a very crucial variable - depending on your workload (one or many
+processes) and cache sizes you might want to experiment with it via
+*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is
+performed a minor collection. Freed objects are no longer referencable and
+just die, just by not being referenced any more; on the other hand, objects
+found to still be alive must survive and are copied from the nursery
+to the old generation. Either to arenas, which are collections
+of objects of the same size, or directly allocated with malloc if they're
+larger.  (A third category, the very large objects, are initially allocated
+outside the nursery and never move.)
+
+Since Incminimark is an incremental GC, the major collection is incremental,
+meaning there should not be any pauses longer than 1ms.
+
+
+Fragmentation
+-------------
+
+Before we discuss issues of "fragmentation", we need a bit of precision.
+There are two kinds of related but distinct issues:
+
+* If the program allocates a lot of memory, and then frees it all by
+  dropping all references to it, then we might expect to see the RSS
+  to drop.  (RSS = Resident Set Size on Linux, as seen by "top"; it is an
+  approximation of the actual memory usage from the OS's point of view.)
+  This might not occur: the RSS may remain at its highest value.  This
+  issue is more precisely caused by the process not returning "free"
+  memory to the OS.  We call this case "unreturned memory".
+
+* After doing the above, if the RSS didn't go down, then at least future
+  allocations should not cause the RSS to grow more.  That is, the process
+  should reuse unreturned memory as long as it has got some left.  If this
+  does not occur, the RSS grows even larger and we have real fragmentation
+  issues.
+
+
+gc.get_stats
+------------
+
+There is a special function in the ``gc`` module called
+``get_stats(memory_pressure=False)``.
+
+``memory_pressure`` controls whether or not to report memory pressure from
+objects allocated outside of the GC, which requires walking the entire heap,
+so it's disabled by default due to its cost. Enable it when debugging
+mysterious memory disappearance.
+
+Example call looks like that::
+    
+    >>> gc.get_stats(True)
+    Total memory consumed:
+    GC used:            4.2MB (peak: 4.2MB)
+       in arenas:            763.7kB
+       rawmalloced:          383.1kB
+       nursery:              3.1MB
+    raw assembler used: 0.0kB
+    memory pressure:    0.0kB
+    -----------------------------
+    Total:              4.2MB
+
+    Total memory allocated:
+    GC allocated:            4.5MB (peak: 4.5MB)
+       in arenas:            763.7kB
+       rawmalloced:          383.1kB
+       nursery:              3.1MB
+    raw assembler allocated: 0.0kB
+    memory pressure:    0.0kB
+    -----------------------------
+    Total:                   4.5MB
+    
+In this particular case, which is just at startup, GC consumes relatively
+little memory and there is even less unused, but allocated memory. In case
+there is a lot of unreturned memory or actual fragmentation, the "allocated"
+can be much higher than "used".  Generally speaking, "peak" will more closely
+resemble the actual memory consumed as reported by RSS.  Indeed, returning
+memory to the OS is a hard and not solved problem.  In PyPy, it occurs only if
+an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each.
+It is also rare for the "rawmalloced" category, at least for common system
+implementations of ``malloc()``.
+
+The details of various fields:
+
+* GC in arenas - small old objects held in arenas. If the amount "allocated"
+  is much higher than the amount "used", we have unreturned memory.  It is
+  possible but unlikely that we have internal fragmentation here.  However,
+  this unreturned memory cannot be reused for any ``malloc()``, including the
+  memory from the "rawmalloced" section.
+
+* GC rawmalloced - large objects allocated with malloc.  This is gives the
+  current (first block of text) and peak (second block of text) memory
+  allocated with ``malloc()``.  The amount of unreturned memory or
+  fragmentation caused by ``malloc()`` cannot easily be reported.  Usually
+  you can guess there is some if the RSS is much larger than the total
+  memory reported for "GC allocated", but do keep in mind that this total
+  does not include malloc'ed memory not known to PyPy's GC at all.  If you
+  guess there is some, consider using `jemalloc`_ as opposed to system malloc.
+
+.. _`jemalloc`: http://jemalloc.net/
+
+* nursery - amount of memory allocated for nursery, fixed at startup,
+  controlled via an environment variable
+
+* raw assembler allocated - amount of assembler memory that JIT feels
+  responsible for
+
+* memory pressure, if asked for - amount of memory we think got allocated
+  via external malloc (eg loading cert store in SSL contexts) that is kept
+  alive by GC objects, but not accounted in the GC
+
 
 .. _minimark-environment-variables:
 
-Minimark
---------
+Environment variables
+---------------------
 
 PyPy's default ``incminimark`` garbage collector is configurable through
 several environment variables:
 
 ``PYPY_GC_NURSERY``
     The nursery size.
-    Defaults to 1/2 of your cache or ``4M``.
+    Defaults to 1/2 of your last-level cache, or ``4M`` if unknown.
     Small values (like 1 or 1KB) are useful for debugging.
 
 ``PYPY_GC_NURSERY_DEBUG``
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -2,9 +2,10 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr,
-    decref)
+    decref, as_pyobj, incref)
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct,
-    PyObjectFields, cts, parse_dir, bootstrap_function, slot_function)
+    PyObjectFields, cts, parse_dir, bootstrap_function, slot_function,
+    Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.import_ import PyImport_Import
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.interpreter.error import OperationError
@@ -31,6 +32,10 @@
     w_type = space.getattr(w_datetime, space.newtext("date"))
     datetimeAPI.c_DateType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
+    # convenient place to modify this, needed since the make_typedescr attach
+    # links the "wrong" struct to W_DateTime_Date, which in turn is needed
+    # because datetime, with a tzinfo entry, inherits from date, without one
+    datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO)
 
     w_type = space.getattr(w_datetime, space.newtext("datetime"))
     datetimeAPI.c_DateTimeType = rffi.cast(
@@ -128,6 +133,7 @@
     # W_DateTime_Date->tp_dealloc
     make_typedescr(W_DateTime_Date.typedef,
                    basestruct=PyDateTime_DateTime.TO,
+                   attach=type_attach,
                    dealloc=date_dealloc,
                   )
 
@@ -138,8 +144,10 @@
 
 def type_attach(space, py_obj, w_obj, w_userdata=None):
     '''Fills a newly allocated py_obj from the w_obj
-       Can be called with a datetime, or a time
     '''
+    if space.type(w_obj).name == 'date':
+        # No tzinfo
+        return
     py_datetime = rffi.cast(PyDateTime_Time, py_obj)
     w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
     if space.is_none(w_tzinfo):
diff --git a/pypy/module/cpyext/test/test_datetime.py 
b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -82,16 +82,6 @@
         date = datetime.datetime.fromtimestamp(0)
         assert space.unwrap(space.str(w_date)) == str(date)
 
-    def test_tzinfo(self, space):
-        w_tzinfo = space.appexec(
-            [], """():
-            from datetime import tzinfo
-            return tzinfo()
-        """)
-        assert PyTZInfo_Check(space, w_tzinfo)
-        assert PyTZInfo_CheckExact(space, w_tzinfo)
-        assert not PyTZInfo_Check(space, space.w_None)
-
 class AppTestDatetime(AppTestCpythonExtensionBase):
     def test_CAPI(self):
         module = self.import_extension('foo', [
@@ -272,3 +262,81 @@
             2000, 6, 6, 6, 6, 6, 6)
         assert module.test_time_macros() == datetime.time(6, 6, 6, 6)
         assert module.test_delta_macros() == datetime.timedelta(6, 6, 6)
+
+    def test_tzinfo(self):
+        module = self.import_extension('foo', [
+            ("time_with_tzinfo", "METH_O",
+             """ PyDateTime_IMPORT;
+                 return PyDateTimeAPI->Time_FromTime(
+                    6, 6, 6, 6, args, PyDateTimeAPI->TimeType);
+             """),
+            ("datetime_with_tzinfo", "METH_O",
+             """ 
+                 PyObject * obj;
+                 int tzrefcnt = args->ob_refcnt; 
+                 PyDateTime_IMPORT;
+                 obj = PyDateTimeAPI->DateTime_FromDateAndTime(
+                    2000, 6, 6, 6, 6, 6, 6, args,
+                    PyDateTimeAPI->DateTimeType);
+                if (!((PyDateTime_DateTime*)obj)->hastzinfo)
+                {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError, "missing tzinfo");
+                    return NULL;
+                }
+                if (((PyDateTime_DateTime*)obj)->tzinfo->ob_refcnt <= tzrefcnt)
+                {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError, "tzinfo refcnt not 
incremented");
+                    return NULL;
+                }
+                return obj;
+                
+             """),
+        ], prologue='#include "datetime.h"\n')
+        from datetime import tzinfo, datetime, timedelta, time
+        # copied from datetime documentation
+        class GMT1(tzinfo):
+           def utcoffset(self, dt):
+               return timedelta(hours=1) + self.dst(dt)
+           def dst(self, dt):
+               return timedelta(0)
+           def tzname(self,dt):
+                return "GMT +1"
+        gmt1 = GMT1()
+        dt1 = module.time_with_tzinfo(gmt1)
+        assert dt1 == time(6, 6, 6, 6, gmt1)
+        assert '+01' in str(dt1)
+        assert module.datetime_with_tzinfo(gmt1) == datetime(
+            2000, 6, 6, 6, 6, 6, 6, gmt1)
+
+    def test_checks(self):
+        module = self.import_extension('foo', [
+            ("checks", "METH_O",
+             """ PyDateTime_IMPORT;
+                 return PyTuple_Pack(10,
+                    PyBool_FromLong(PyDateTime_Check(args)),
+                    PyBool_FromLong(PyDateTime_CheckExact(args)),
+                    PyBool_FromLong(PyDate_Check(args)),
+                    PyBool_FromLong(PyDate_CheckExact(args)),
+                    PyBool_FromLong(PyTime_Check(args)),
+                    PyBool_FromLong(PyTime_CheckExact(args)),
+                    PyBool_FromLong(PyDelta_Check(args)),
+                    PyBool_FromLong(PyDelta_CheckExact(args)),
+                    PyBool_FromLong(PyTZInfo_Check(args)),
+                    PyBool_FromLong(PyTZInfo_CheckExact(args))
+                );
+             """),
+        ], prologue='#include "datetime.h"\n')
+        from datetime import tzinfo, datetime, timedelta, time, date
+        o = date(1, 1, 1)
+        assert module.checks(o) == (False,) * 2 + (True,) * 2 + (False,) * 6
+        o = time(1, 1, 1)
+        assert module.checks(o) == (False,) * 4 + (True,) * 2 + (False,) * 4
+        o = timedelta(1, 1, 1)
+        assert module.checks(o) == (False,) * 6 + (True,) * 2 + (False,) * 2
+        o = datetime(1, 1, 1)
+        assert module.checks(o) == (True,) * 3 + (False,) * 7 # 
isinstance(datetime, date)
+        o = tzinfo()
+        assert module.checks(o) == (False,) * 8 + (True,) * 2
+        
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -72,7 +72,7 @@
 
     def __repr__(self):
         if self._s.total_memory_pressure != -1:
-            extra = "\nmemory pressure:    %s" % self.total_memory_pressure
+            extra = "\n    memory pressure:    %s" % self.total_memory_pressure
         else:
             extra = ""
         return """Total memory consumed:
@@ -109,5 +109,5 @@
            self.memory_allocated_sum)
 
 
-def get_stats():
-    return GcStats(gc._get_stats())
+def get_stats(memory_pressure=False):
+    return GcStats(gc._get_stats(memory_pressure=memory_pressure))
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

Reply via email to