Author: Carl Friedrich Bolz-Tereick <cfb...@gmx.de>
Branch: guard-compatible
Changeset: r93903:047c33d38a20
Date: 2018-02-22 20:33 +0100
http://bitbucket.org/pypy/pypy/changeset/047c33d38a20/
Log: merge default
diff too long, truncating to 2000 out of 10294 lines
diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py
b/extra_tests/test_pyrepl/__init__.py
rename from pypy/module/test_lib_pypy/pyrepl/__init__.py
rename to extra_tests/test_pyrepl/__init__.py
--- a/pypy/module/test_lib_pypy/pyrepl/__init__.py
+++ b/extra_tests/test_pyrepl/__init__.py
@@ -1,3 +1,1 @@
-import sys
-import lib_pypy.pyrepl
-sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl']
+
diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py
b/extra_tests/test_pyrepl/infrastructure.py
rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py
rename to extra_tests/test_pyrepl/infrastructure.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py
b/extra_tests/test_pyrepl/test_basic.py
rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py
rename to extra_tests/test_pyrepl/test_basic.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py
b/extra_tests/test_pyrepl/test_bugs.py
rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py
rename to extra_tests/test_pyrepl/test_bugs.py
diff --git a/extra_tests/test_pyrepl/test_functional.py
b/extra_tests/test_pyrepl/test_functional.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_pyrepl/test_functional.py
@@ -0,0 +1,28 @@
+# Copyright 2000-2007 Michael Hudson-Doyle <mica...@gmail.com>
+# Maciek Fijalkowski
+# License: MIT
+# some functional tests, to see if this is really working
+
+import pytest
+import sys
+
+
+@pytest.fixture()
+def child():
+ try:
+ import pexpect
+ except ImportError:
+ pytest.skip("no pexpect module")
+ except SyntaxError:
+ pytest.skip('pexpect wont work on py3k')
+ child = pexpect.spawn(sys.executable, ['-S'], timeout=10)
+ child.logfile = sys.stdout
+ child.sendline('from pyrepl.python_reader import main')
+ child.sendline('main()')
+ return child
+
+
+def test_basic(child):
+ child.sendline('a = 3')
+ child.sendline('a')
+ child.expect('3')
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py
b/extra_tests/test_pyrepl/test_keymap.py
rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py
rename to extra_tests/test_pyrepl/test_keymap.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py
b/extra_tests/test_pyrepl/test_reader.py
rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py
rename to extra_tests/test_pyrepl/test_reader.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py
b/extra_tests/test_pyrepl/test_readline.py
rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py
rename to extra_tests/test_pyrepl/test_readline.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py
b/extra_tests/test_pyrepl/test_wishes.py
rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py
rename to extra_tests/test_pyrepl/test_wishes.py
diff --git a/get_externals.py b/get_externals.py
new file mode 100644
--- /dev/null
+++ b/get_externals.py
@@ -0,0 +1,69 @@
+'''Get external dependencies for building PyPy
+they will end up in the platform.host().basepath, something like
repo-root/external
+'''
+
+from __future__ import print_function
+
+import argparse
+import os
+import zipfile
+from subprocess import Popen, PIPE
+from rpython.translator.platform import host
+
+def runcmd(cmd, verbose):
+ stdout = stderr = ''
+ report = False
+ try:
+ p = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = p.communicate()
+ if p.wait() != 0 or verbose:
+ report = True
+ except Exception as e:
+ stderr = str(e) + '\n' + stderr
+ report = True
+ if report:
+ print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout,
stderr))
+ if stderr:
+ raise RuntimeError(stderr)
+
+def checkout_repo(dest='externals', org='pypy', branch='default',
verbose=False):
+ url = 'https://bitbucket.org/{}/externals'.format(org)
+ if not os.path.exists(dest):
+ cmd = ['hg','clone',url,dest]
+ runcmd(cmd, verbose)
+ cmd = ['hg','-R', dest, 'update',branch]
+ runcmd(cmd, verbose)
+
+def extract_zip(externals_dir, zip_path):
+ with zipfile.ZipFile(os.fspath(zip_path)) as zf:
+ zf.extractall(os.fspath(externals_dir))
+ return externals_dir / zf.namelist()[0].split('/')[0]
+
+def parse_args():
+ p = argparse.ArgumentParser()
+ p.add_argument('-v', '--verbose', action='store_true')
+ p.add_argument('-O', '--organization',
+ help='Organization owning the deps repos', default='pypy')
+ p.add_argument('-e', '--externals', default=host.externals,
+ help='directory in which to store dependencies',
+ )
+ p.add_argument('-b', '--branch', default=host.externals_branch,
+ help='branch to check out',
+ )
+ p.add_argument('-p', '--platform', default=None,
+ help='someday support cross-compilation, ignore for now',
+ )
+ return p.parse_args()
+
+
+def main():
+ args = parse_args()
+ checkout_repo(
+ dest=args.externals,
+ org=args.organization,
+ branch=args.branch,
+ verbose=args.verbose,
+ )
+
+if __name__ == '__main__':
+ main()
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -153,9 +153,10 @@
factory = Connection if not factory else factory
# an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
# backed by :memory: or a file)
+ res = factory(database, timeout, detect_types, isolation_level,
+ check_same_thread, factory, cached_statements)
add_memory_pressure(100 * 1024)
- return factory(database, timeout, detect_types, isolation_level,
- check_same_thread, factory, cached_statements)
+ return res
def _unicode_text_factory(x):
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,11 +1,12 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.11.3
+Version: 1.11.4
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
Author-email: python-c...@googlegroups.com
License: MIT
+Description-Content-Type: UNKNOWN
Description:
CFFI
====
@@ -27,5 +28,7 @@
Classifier: Programming Language :: Python :: 3.2
Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI
from .error import CDefError, FFIError, VerificationError, VerificationMissing
-__version__ = "1.11.3"
-__version_info__ = (1, 11, 3)
+__version__ = "1.11.4"
+__version_info__ = (1, 11, 4)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,37 +8,20 @@
the same works for the other two macros. Py_DEBUG implies them,
but not the other way around.
- Issue #350: more mess: on Windows, with _MSC_VER, we have to define
- Py_LIMITED_API even before including pyconfig.h. In that case, we
- guess what pyconfig.h will do to the macros above, and check our
- guess after the #include.
+ Issue #350 is still open: on Windows, the code here causes it to link
+ with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was
+ attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
+ does not make PYTHON3.DLL available, and so the "correctly" compiled
+ version would not run inside a virtualenv. We will re-apply the fix
+ after virtualenv has been fixed for some time. For explanation, see
+ issue #355. For a workaround if you want PYTHON3.DLL and don't worry
+ about virtualenv, see issue #350. See also 'py_limited_api' in
+ setuptools_ext.py.
*/
#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-# ifdef _MSC_VER
-# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) &&
!defined(Py_REF_DEBUG)
-# define Py_LIMITED_API
-# endif
-# include <pyconfig.h>
- /* sanity-check: Py_LIMITED_API will cause crashes if any of these
- are also defined. Normally, the Python file PC/pyconfig.h does not
- cause any of these to be defined, with the exception that _DEBUG
- causes Py_DEBUG. Double-check that. */
-# ifdef Py_LIMITED_API
-# if defined(Py_DEBUG)
-# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set"
-# endif
-# if defined(Py_TRACE_REFS)
-# error "pyconfig.h unexpectedly defines Py_TRACE_REFS"
-# endif
-# if defined(Py_REF_DEBUG)
-# error "pyconfig.h unexpectedly defines Py_REF_DEBUG"
-# endif
-# endif
-# else
-# include <pyconfig.h>
-# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-# define Py_LIMITED_API
-# endif
+# include <pyconfig.h>
+# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+# define Py_LIMITED_API
# endif
#endif
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -247,7 +247,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.11.3"
+ "\ncompiled with cffi version: 1.11.4"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -17,10 +17,13 @@
"""
from __future__ import division
-import time as _time
+import time as _timemodule
import math as _math
import struct as _struct
+# for cpyext, use these as base classes
+from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
+
_SENTINEL = object()
def _cmp(x, y):
@@ -179,7 +182,7 @@
def _build_struct_time(y, m, d, hh, mm, ss, dstflag):
wday = (_ymd2ord(y, m, d) + 6) % 7
dnum = _days_before_month(y, m) + d
- return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag))
+ return _timemodule.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag))
def _format_time(hh, mm, ss, us):
# Skip trailing microseconds when us==0.
@@ -247,7 +250,7 @@
else:
push(ch)
newformat = "".join(newformat)
- return _time.strftime(newformat, timetuple)
+ return _timemodule.strftime(newformat, timetuple)
# Just raise TypeError if the arg isn't None or a string.
def _check_tzname(name):
@@ -433,7 +436,7 @@
raise TypeError("unsupported type for timedelta %s component: %s" %
(tag, type(num)))
-class timedelta(object):
+class timedelta(deltainterop):
"""Represent the difference between two datetime objects.
Supported operators:
@@ -489,7 +492,7 @@
if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS:
raise OverflowError("days=%d; must have magnitude <= %d" % (d,
_MAX_DELTA_DAYS))
- self = object.__new__(cls)
+ self = deltainterop.__new__(cls)
self._days = d
self._seconds = s
self._microseconds = us
@@ -667,7 +670,7 @@
timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1)
timedelta.resolution = timedelta(microseconds=1)
-class date(object):
+class date(dateinterop):
"""Concrete date type.
Constructors:
@@ -707,12 +710,12 @@
if month is None and isinstance(year, bytes) and len(year) == 4 and \
1 <= ord(year[2]) <= 12:
# Pickle support
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self.__setstate(year)
self._hashcode = -1
return self
year, month, day = _check_date_fields(year, month, day)
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self._year = year
self._month = month
self._day = day
@@ -724,13 +727,13 @@
@classmethod
def fromtimestamp(cls, t):
"Construct a date from a POSIX timestamp (like time.time())."
- y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)
+ y, m, d, hh, mm, ss, weekday, jday, dst = _timemodule.localtime(t)
return cls(y, m, d)
@classmethod
def today(cls):
"Construct a date from time.time()."
- t = _time.time()
+ t = _timemodule.time()
return cls.fromtimestamp(t)
@classmethod
@@ -1061,7 +1064,7 @@
_tzinfo_class = tzinfo
-class time(object):
+class time(timeinterop):
"""Time with time zone.
Constructors:
@@ -1097,14 +1100,14 @@
"""
if isinstance(hour, bytes) and len(hour) == 6 and ord(hour[0]) < 24:
# Pickle support
- self = object.__new__(cls)
+ self = timeinterop.__new__(cls)
self.__setstate(hour, minute or None)
self._hashcode = -1
return self
hour, minute, second, microsecond = _check_time_fields(
hour, minute, second, microsecond)
_check_tzinfo_arg(tzinfo)
- self = object.__new__(cls)
+ self = timeinterop.__new__(cls)
self._hour = hour
self._minute = minute
self._second = second
@@ -1408,15 +1411,20 @@
if isinstance(year, bytes) and len(year) == 10 and \
1 <= ord(year[2]) <= 12:
# Pickle support
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self.__setstate(year, month)
self._hashcode = -1
return self
- year, month, day = _check_date_fields(year, month, day)
- hour, minute, second, microsecond = _check_time_fields(
- hour, minute, second, microsecond)
+ elif isinstance(year, tuple) and len(year) == 7:
+ # Used by internal functions where the arguments are guaranteed to
+ # be valid.
+ year, month, day, hour, minute, second, microsecond = year
+ else:
+ year, month, day = _check_date_fields(year, month, day)
+ hour, minute, second, microsecond = _check_time_fields(
+ hour, minute, second, microsecond)
_check_tzinfo_arg(tzinfo)
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self._year = year
self._month = month
self._day = day
@@ -1461,7 +1469,7 @@
A timezone info object may be passed in as well.
"""
_check_tzinfo_arg(tz)
- converter = _time.localtime if tz is None else _time.gmtime
+ converter = _timemodule.localtime if tz is None else _timemodule.gmtime
self = cls._from_timestamp(converter, timestamp, tz)
if tz is not None:
self = tz.fromutc(self)
@@ -1470,7 +1478,7 @@
@classmethod
def utcfromtimestamp(cls, t):
"Construct a UTC datetime from a POSIX timestamp (like time.time())."
- return cls._from_timestamp(_time.gmtime, t, None)
+ return cls._from_timestamp(_timemodule.gmtime, t, None)
@classmethod
def _from_timestamp(cls, converter, timestamp, tzinfo):
@@ -1488,18 +1496,18 @@
us = 0
y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp)
ss = min(ss, 59) # clamp out leap seconds if the platform has them
- return cls(y, m, d, hh, mm, ss, us, tzinfo)
+ return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo)
@classmethod
def now(cls, tz=None):
"Construct a datetime from time.time() and optional time zone info."
- t = _time.time()
+ t = _timemodule.time()
return cls.fromtimestamp(t, tz)
@classmethod
def utcnow(cls):
"Construct a UTC datetime from time.time()."
- t = _time.time()
+ t = _timemodule.time()
return cls.utcfromtimestamp(t)
@classmethod
@@ -1797,7 +1805,7 @@
return diff and 1 or 0
def _add_timedelta(self, other, factor):
- y, m, d, hh, mm, ss, us = _normalize_datetime(
+ result = _normalize_datetime(
self._year,
self._month,
self._day + other.days * factor,
@@ -1805,7 +1813,7 @@
self._minute,
self._second + other.seconds * factor,
self._microsecond + other.microseconds * factor)
- return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo)
+ return datetime(result, tzinfo=self._tzinfo)
def __add__(self, other):
"Add a datetime and a timedelta."
diff --git a/lib_pypy/dbm.py b/lib_pypy/dbm.py
--- a/lib_pypy/dbm.py
+++ b/lib_pypy/dbm.py
@@ -157,7 +157,14 @@
def open(filename, flag='r', mode=0666):
"open a DBM database"
if not isinstance(filename, str):
- raise TypeError("expected string")
+ if sys.version_info < (3,) and isinstance(filename, unicode):
+ # unlike CPython we'll encode 'filename' with filesystemencoding
+ # instead of defaultencoding, because that seems like a far
+ # better idea. But I'm also open for saying that we should
+ # rather go for bug-to-bug compatibility instead.
+ filename = filename.encode(sys.getfilesystemencoding())
+ else:
+ raise TypeError("expected string")
openflag = 0
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
Metadata-Version: 1.0
Name: greenlet
-Version: 0.4.12
+Version: 0.4.13
Summary: Lightweight in-process concurrent programming
Home-page: https://github.com/python-greenlet/greenlet
Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
import sys
import _continuation
-__version__ = "0.4.12"
+__version__ = "0.4.13"
# ____________________________________________________________
# Exceptions
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -1,17 +1,137 @@
-Garbage collector configuration
-===============================
+Garbage collector documentation and configuration
+=================================================
+
+
+Incminimark
+-----------
+
+PyPy's default garbage collector is called incminimark - it's an incremental,
+generational moving collector. Here we hope to explain a bit how it works
+and how it can be tuned to suit the workload.
+
+Incminimark first allocates objects in so called *nursery* - place for young
+objects, where allocation is very cheap, being just a pointer bump. The nursery
+size is a very crucial variable - depending on your workload (one or many
+processes) and cache sizes you might want to experiment with it via
+*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is
+performed a minor collection. Freed objects are no longer referencable and
+just die, just by not being referenced any more; on the other hand, objects
+found to still be alive must survive and are copied from the nursery
+to the old generation. Either to arenas, which are collections
+of objects of the same size, or directly allocated with malloc if they're
+larger. (A third category, the very large objects, are initially allocated
+outside the nursery and never move.)
+
+Since Incminimark is an incremental GC, the major collection is incremental,
+meaning there should not be any pauses longer than 1ms.
+
+
+Fragmentation
+-------------
+
+Before we discuss issues of "fragmentation", we need a bit of precision.
+There are two kinds of related but distinct issues:
+
+* If the program allocates a lot of memory, and then frees it all by
+ dropping all references to it, then we might expect to see the RSS
+ to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an
+ approximation of the actual memory usage from the OS's point of view.)
+ This might not occur: the RSS may remain at its highest value. This
+ issue is more precisely caused by the process not returning "free"
+ memory to the OS. We call this case "unreturned memory".
+
+* After doing the above, if the RSS didn't go down, then at least future
+ allocations should not cause the RSS to grow more. That is, the process
+ should reuse unreturned memory as long as it has got some left. If this
+ does not occur, the RSS grows even larger and we have real fragmentation
+ issues.
+
+
+gc.get_stats
+------------
+
+There is a special function in the ``gc`` module called
+``get_stats(memory_pressure=False)``.
+
+``memory_pressure`` controls whether or not to report memory pressure from
+objects allocated outside of the GC, which requires walking the entire heap,
+so it's disabled by default due to its cost. Enable it when debugging
+mysterious memory disappearance.
+
+Example call looks like that::
+
+ >>> gc.get_stats(True)
+ Total memory consumed:
+ GC used: 4.2MB (peak: 4.2MB)
+ in arenas: 763.7kB
+ rawmalloced: 383.1kB
+ nursery: 3.1MB
+ raw assembler used: 0.0kB
+ memory pressure: 0.0kB
+ -----------------------------
+ Total: 4.2MB
+
+ Total memory allocated:
+ GC allocated: 4.5MB (peak: 4.5MB)
+ in arenas: 763.7kB
+ rawmalloced: 383.1kB
+ nursery: 3.1MB
+ raw assembler allocated: 0.0kB
+ memory pressure: 0.0kB
+ -----------------------------
+ Total: 4.5MB
+
+In this particular case, which is just at startup, GC consumes relatively
+little memory and there is even less unused, but allocated memory. In case
+there is a lot of unreturned memory or actual fragmentation, the "allocated"
+can be much higher than "used". Generally speaking, "peak" will more closely
+resemble the actual memory consumed as reported by RSS. Indeed, returning
+memory to the OS is a hard and not solved problem. In PyPy, it occurs only if
+an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each.
+It is also rare for the "rawmalloced" category, at least for common system
+implementations of ``malloc()``.
+
+The details of various fields:
+
+* GC in arenas - small old objects held in arenas. If the amount "allocated"
+ is much higher than the amount "used", we have unreturned memory. It is
+ possible but unlikely that we have internal fragmentation here. However,
+ this unreturned memory cannot be reused for any ``malloc()``, including the
+ memory from the "rawmalloced" section.
+
+* GC rawmalloced - large objects allocated with malloc. This is gives the
+ current (first block of text) and peak (second block of text) memory
+ allocated with ``malloc()``. The amount of unreturned memory or
+ fragmentation caused by ``malloc()`` cannot easily be reported. Usually
+ you can guess there is some if the RSS is much larger than the total
+ memory reported for "GC allocated", but do keep in mind that this total
+ does not include malloc'ed memory not known to PyPy's GC at all. If you
+ guess there is some, consider using `jemalloc`_ as opposed to system malloc.
+
+.. _`jemalloc`: http://jemalloc.net/
+
+* nursery - amount of memory allocated for nursery, fixed at startup,
+ controlled via an environment variable
+
+* raw assembler allocated - amount of assembler memory that JIT feels
+ responsible for
+
+* memory pressure, if asked for - amount of memory we think got allocated
+ via external malloc (eg loading cert store in SSL contexts) that is kept
+ alive by GC objects, but not accounted in the GC
+
.. _minimark-environment-variables:
-Minimark
---------
+Environment variables
+---------------------
PyPy's default ``incminimark`` garbage collector is configurable through
several environment variables:
``PYPY_GC_NURSERY``
The nursery size.
- Defaults to 1/2 of your cache or ``4M``.
+ Defaults to 1/2 of your last-level cache, or ``4M`` if unknown.
Small values (like 1 or 1KB) are useful for debugging.
``PYPY_GC_NURSERY_DEBUG``
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -1,26 +1,41 @@
Potential Project List
======================
-Google Summer of Code 2017
---------------------------
+Getting involved
+----------------
-PyPy is generally open to new ideas for Google Summer of Code. We are happy to
accept good ideas around the PyPy ecosystem. If you need more information about
the ideas we propose for this year please join us on irc, channel #pypy
(freenode). If you are unsure, but still think that you can make a valuable
contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing
list.
-
+We are happy to discuss ideas around the PyPy ecosystem.
+If you are interested in palying with RPython or PyPy, or have a new idea not
+mentioned here please join us on irc, channel #pypy (freenode). If you are
unsure,
+but still think that you can make a valuable contribution to PyPy, dont
+hesitate to contact us on #pypy or on our mailing list. Here are some ideas
+to get you thinking:
* **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than
CPython.
- Two examples: 1) PyPy seems to allocate and keep alive more strings when
importing a big Python modules.
- 2) The base interpreter size (cold VM started from a console) of PyPy is
bigger than the one of CPython.
- The general procedure of this project is: Run both CPython and PyPy of the
same Python version and
- compare the memory usage (using Massif or other tools).
+ Two examples: 1) PyPy seems to allocate and keep alive more strings when
+ importing a big Python modules. 2) The base interpreter size (cold VM
started
+ from a console) of PyPy is bigger than the one of CPython. The general
+ procedure of this project is: Run both CPython and PyPy of the same Python
+ version and compare the memory usage (using Massif or other tools).
If PyPy consumes a lot more memory then find and resolve the issue.
-* **VMProf + memory profiler**: vmprof by now has a memory profiler that can
be used already. We want extend it with more features and resolve some current
limitations.
+* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We
+ want extend it with new features and resolve some current limitations.
-* **VMProf visualisations**: vmprof just shows a flame graph of the
statistical profile and some more information about specific call sites. It
would be very interesting to experiment with different information (such as
memory, or even information generated by our jit compiler).
+* **VMProf visualisations**: vmprof shows a flame graph of the statistical
+ profile and some more information about specific call sites. It would be
+ very interesting to experiment with different information (such as memory,
+ or even information generated by our jit compiler).
-* **Explicit typing in RPython**: PyPy wants to have better ways to specify
the signature and class attribute types in RPython. See more information about
this topic below on this page.
+* **Explicit typing in RPython**: PyPy wants to have better ways to specify
+ the signature and class attribute types in RPython. See more information
+ about this topic below on this page.
-* **Virtual Reality (VR) visualisations for vmprof**: This is a very open
topic with lots of freedom to explore data visualisation for profiles. No VR
hardware would be needed for this project. Either universities provide such
hardware or in any other case we potentially can lend the VR hardware setup.
+* **Virtual Reality (VR) visualisations for vmprof**: This is a very open
+ topic with lots of freedom to explore data visualisation for profiles. No
+ VR hardware would be needed for this project. Either universities provide
+ such hardware or in any other case we potentially can lend the VR hardware
+ setup.
Simple tasks for newcomers
--------------------------
@@ -34,6 +49,11 @@
* Implement AF_XXX packet types of sockets:
https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets
+* Help with documentation. One task would be to document rpython configuration
+ options currently listed only on :doc:`this site <configuration>` also on the
+ RPython_ documentation site.
+
+.. _RPython: http://rpython.readthedocs.io
Mid-to-large tasks
------------------
@@ -201,7 +221,9 @@
Introduce new benchmarks
------------------------
-We're usually happy to introduce new benchmarks. Please consult us
+Our benchmark runner_ is showing its age. We should merge with the `CPython
site`_
+
+Additionally, we're usually happy to introduce new benchmarks. Please consult
us
before, but in general something that's real-world python code
and is not already represented is welcome. We need at least a standalone
script that can run without parameters. Example ideas (benchmarks need
@@ -209,6 +231,8 @@
* `hg`
+.. _runner: http://speed.pypy.org
+.. _`CPython site`: https://speed.python.org/
======================================
Make more python modules pypy-friendly
@@ -238,15 +262,6 @@
using more pypy-friendly technologies, e.g. cffi. Here is a partial list of
good work that needs to be finished:
-**matplotlib** https://github.com/matplotlib/matplotlib
-
- Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of
- matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the
- tkagg backend can function.
-
- TODO: the matplotlib branch passes numpy arrays by value (copying all the
- data), this proof-of-concept needs help to become completely compliant
-
**wxPython** https://bitbucket.org/amauryfa/wxpython-cffi
Status: A project by a PyPy developer to adapt the Phoenix sip build
system to cffi
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -10,3 +10,37 @@
Big refactoring of some cpyext code, which avoids a lot of nonsense when
calling C from Python and vice-versa: the result is a big speedup in
function/method calls, up to 6 times faster.
+
+.. branch: cpyext-datetime2
+
+Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD
+
+
+.. branch: mapdict-size-limit
+
+Fix a corner case of mapdict: When an instance is used like a dict (using
+``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are
+added, then the performance using mapdict is linear in the number of
+attributes. This is now fixed (by switching to a regular dict after 80
+attributes).
+
+
+.. branch: cpyext-faster-arg-passing
+
+When using cpyext, improve the speed of passing certain objects from PyPy to C
+code, most notably None, True, False, types, all instances of C-defined types.
+Before, a dict lookup was needed every time such an object crossed over, now it
+is just a field read.
+
+
+.. branch: 2634_datetime_timedelta_performance
+
+Improve datetime + timedelta performance.
+
+.. branch: memory-accounting
+
+Improve way to describe memory
+
+.. branch: msvc14
+
+Allow compilaiton with Visual Studio 2017 compiler suite on windows
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -39,10 +39,24 @@
.. _Microsoft Visual C++ Compiler for Python 2.7:
https://www.microsoft.com/en-us/download/details.aspx?id=44266
+Installing "Build Tools for Visual Studio 2017" (for Python 3)
+--------------------------------------------------------------
+
+As documented in the CPython Wiki_, CPython now recommends Visual C++ version
+14.0. A compact version of the compiler suite can be obtained from Microsoft_
+downloads, search the page for "Build Tools for Visual Studio 2017".
+
+You will also need to install the the `Windows SDK`_ in order to use the
+`mt.exe` mainfest compiler.
+
+.. _Wiki: https://wiki.python.org/moin/WindowsCompilers
+.. _Microsoft: https://www.visualstudio.com/downloads
+.. _`Windows SDK`:
https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk
+
Translating PyPy with Visual Studio
-----------------------------------
-We routinely test translation using v9, also known as Visual Studio 2008.
+We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14.
Other configurations may work as well.
The translation scripts will set up the appropriate environment variables
@@ -82,8 +96,8 @@
.. _build instructions: http://pypy.org/download.html#building-from-source
-Setting Up Visual Studio for building SSL in Python3
-----------------------------------------------------
+Setting Up Visual Studio 9.0 for building SSL in Python3
+--------------------------------------------------------
On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step
after
translation. However ``distutils`` does not support the Micorosft-provided
Visual C
@@ -132,243 +146,14 @@
Installing external packages
----------------------------
-On Windows, there is no standard place where to download, build and
-install third-party libraries. We recommend installing them in the parent
-directory of the pypy checkout. For example, if you installed pypy in
-``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``. You must then set the
-INCLUDE, LIB and PATH (for DLLs) environment variables appropriately.
+We uses a `repository` parallel to pypy to hold binary compiled versions of the
+build dependencies for windows. As part of the `rpython` setup stage,
environment
+variables will be set to use these dependencies. The repository has a README
+file on how to replicate, and a branch for each supported platform. You may run
+ the `get_externals.py` utility to checkout the proper branch for your platform
+and PyPy version.
-
-Abridged method (using Visual Studio 2008)
-------------------------------------------
-
-Download the versions of all the external packages from
-https://bitbucket.org/pypy/pypy/downloads/local_59.zip
-(for post-5.8 builds) with sha256 checksum
-``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911``
-https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
-(to reproduce 5.8 builds) with sha256 checksum
-``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
-https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
-(for 2.4 release and later) or
-https://bitbucket.org/pypy/pypy/downloads/local.zip
-(for pre-2.4 versions)
-Then expand it into the base directory (base_dir) and modify your environment
-to reflect this::
-
- set PATH=<base_dir>\bin;%PATH%
- set INCLUDE=<base_dir>\include;%INCLUDE%
- set LIB=<base_dir>\lib;%LIB%
-
-Now you should be good to go. If you choose this method, you do not need
-to download/build anything else.
-
-Nonabridged method (building from scratch)
-------------------------------------------
-
-If you want to, you can rebuild everything from scratch by continuing.
-
-
-The Boehm garbage collector
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This library is needed if you plan to use the ``--gc=boehm`` translation
-option (this is the default at some optimization levels like ``-O1``,
-but unneeded for high-performance translations like ``-O2``).
-You may get it at
-http://hboehm.info/gc/gc_source/gc-7.1.tar.gz
-
-Versions 7.0 and 7.1 are known to work; the 6.x series won't work with
-RPython. Unpack this folder in the base directory.
-The default GC_abort(...) function in misc.c will try to open a MessageBox.
-You may want to disable this with the following patch::
-
- --- a/misc.c Sun Apr 20 14:08:27 2014 +0300
- +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300
- @@ -1058,7 +1058,7 @@
- #ifndef PCR
- void GC_abort(const char *msg)
- {
- -# if defined(MSWIN32)
- +# if 0 && defined(MSWIN32)
- (void) MessageBoxA(NULL, msg, "Fatal error in gc",
MB_ICONERROR|MB_OK);
- # else
- GC_err_printf("%s\n", msg);
-
-Then open a command prompt::
-
- cd gc-7.1
- nmake -f NT_THREADS_MAKEFILE
- copy Release\gc.dll <somewhere in the PATH>
-
-
-The zlib compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in
-the base directory. Then compile::
-
- cd zlib-1.2.11
- nmake -f win32\Makefile.msc
- copy zlib.lib <somewhere in LIB>
- copy zlib.h zconf.h <somewhere in INCLUDE>
- copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes)
-
-
-The bz2 compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Get the same version of bz2 used by python and compile as a static library::
-
- svn export http://svn.python.org/projects/external/bzip2-1.0.6
- cd bzip2-1.0.6
- nmake -f makefile.msc
- copy libbz2.lib <somewhere in LIB>
- copy bzlib.h <somewhere in INCLUDE>
-
-
-The sqlite3 database library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi
-wrapper is compiled when the module is imported for the first time.
-The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility.
-
-
-The expat XML parser
-~~~~~~~~~~~~~~~~~~~~
-
-CPython compiles expat from source as part of the build. PyPy uses the same
-code base, but expects to link to a static lib of expat. Here are instructions
-to reproduce the static lib in version 2.2.4.
-
-Download the source code of expat: https://github.com/libexpat/libexpat.
-``git checkout`` the proper tag, in this case ``R_2_2_4``. Run
-``vcvars.bat`` to set up the visual compiler tools, and CD into the source
-directory. Create a file ``stdbool.h`` with the content
-
-.. code-block:: c
-
- #pragma once
-
- #define false 0
- #define true 1
-
- #define bool int
-
-and put it in a place on the ``INCLUDE`` path, or create it in the local
-directory and add ``.`` to the ``INCLUDE`` path::
-
- SET INCLUDE=%INCLUDE%;.
-
-Then compile all the ``*.c`` file into ``*.obj``::
-
- cl.exe /nologo /MD /O2 *c /c
- rem for debug
- cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c
-
-You may need to move some variable declarations to the beginning of the
-function, to be compliant with C89 standard. Here is the diff for version 2.2.4
-
-.. code-block:: diff
-
- diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c
- index 007aed0..a2dcaad 100644
- --- a/expat/lib/xmltok.c
- +++ b/expat/lib/xmltok.c
- @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
- /* Avoid copying partial characters (due to limited space). */
- const ptrdiff_t bytesAvailable = fromLim - *fromP;
- const ptrdiff_t bytesStorable = toLim - *toP;
- + const char * fromLimBefore;
- + ptrdiff_t bytesToCopy;
- if (bytesAvailable > bytesStorable) {
- fromLim = *fromP + bytesStorable;
- output_exhausted = true;
- }
-
- /* Avoid copying partial characters (from incomplete input). */
- - const char * const fromLimBefore = fromLim;
- + fromLimBefore = fromLim;
- align_limit_to_full_utf8_characters(*fromP, &fromLim);
- if (fromLim < fromLimBefore) {
- input_incomplete = true;
- }
-
- - const ptrdiff_t bytesToCopy = fromLim - *fromP;
- + bytesToCopy = fromLim - *fromP;
- memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy);
- *fromP += bytesToCopy;
- *toP += bytesToCopy;
-
-
-Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests)::
-
- cl /LD *.obj libexpat.def /Felibexpat.dll
- rem for debug
- rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll
-
- rem this will override the export library created in the step above
- rem but tests do not need the export library, they load the dll dynamically
- lib *.obj /out:libexpat.lib
-
-Then, copy
-
-- ``libexpat.lib`` into LIB
-- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE
-- ``libexpat.dll`` into PATH
-
-
-The OpenSSL library
-~~~~~~~~~~~~~~~~~~~
-
-OpenSSL needs a Perl interpreter to configure its makefile. You may
-use the one distributed by ActiveState, or the one from cygwin.::
-
- svn export http://svn.python.org/projects/external/openssl-1.0.2k
- cd openssl-1.0.2k
- perl Configure VC-WIN32 no-idea no-mdc2
- ms\do_ms.bat
- nmake -f ms\nt.mak install
- copy out32\*.lib <somewhere in LIB>
- xcopy /S include\openssl <somewhere in INCLUDE>
-
-For tests you will also need the dlls::
- nmake -f ms\ntdll.mak install
- copy out32dll\*.dll <somewhere in PATH>
-
-TkInter module support
-~~~~~~~~~~~~~~~~~~~~~~
-
-Note that much of this is taken from the cpython build process.
-Tkinter is imported via cffi, so the module is optional. To recreate the tcltk
-directory found for the release script, create the dlls, libs, headers and
-runtime by running::
-
- svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
- svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
- cd tcl85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0
INSTALLDIR=..\..\tcltk clean all
- nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
- cd ..\..\tk85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1
INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1
INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
- copy ..\..\tcltk\bin\* <somewhere in PATH>
- copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
- xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
-
-The lzma compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Python 3.3 ship with CFFI wrappers for the lzma library, which can be
-downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version
-5.0.5, a prebuilt version can be downloaded from
-http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature
-http://tukaani.org/xz/xz-5.0.5-windows.zip.sig
-
-Then copy the headers to the include directory, rename ``liblzma.a`` to
-``lzma.lib`` and copy it to the lib directory
-
+.. _repository: https://bitbucket.org/pypy/external
Using the mingw compiler
------------------------
diff --git a/pypy/interpreter/astcompiler/assemble.py
b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,7 +1,7 @@
"""Python control flow graph generation and bytecode assembly."""
+import math
import os
-from rpython.rlib import rfloat
from rpython.rlib.objectmodel import we_are_translated
from pypy.interpreter.astcompiler import ast, misc, symtable
@@ -266,7 +266,7 @@
w_type = space.type(obj)
if space.is_w(w_type, space.w_float):
val = space.float_w(obj)
- if val == 0.0 and rfloat.copysign(1., val) < 0:
+ if val == 0.0 and math.copysign(1., val) < 0:
w_key = space.newtuple([obj, space.w_float, space.w_None])
else:
w_key = space.newtuple([obj, space.w_float])
@@ -276,9 +276,9 @@
real = space.float_w(w_real)
imag = space.float_w(w_imag)
real_negzero = (real == 0.0 and
- rfloat.copysign(1., real) < 0)
+ math.copysign(1., real) < 0)
imag_negzero = (imag == 0.0 and
- rfloat.copysign(1., imag) < 0)
+ math.copysign(1., imag) < 0)
if real_negzero and imag_negzero:
tup = [obj, space.w_complex, space.w_None, space.w_None,
space.w_None]
diff --git a/pypy/interpreter/astcompiler/astbuilder.py
b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1096,17 +1096,21 @@
encoding = self.compile_info.encoding
flags = self.compile_info.flags
unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS
- try:
- sub_strings_w = [parsestring.parsestr(space, encoding,
atom_node.get_child(i).get_value(),
- unicode_literals)
- for i in range(atom_node.num_children())]
- except error.OperationError as e:
- if not e.match(space, space.w_UnicodeError):
- raise
- # UnicodeError in literal: turn into SyntaxError
- e.normalize_exception(space)
- errmsg = space.text_w(space.str(e.get_w_value(space)))
- raise self.error('(unicode error) %s' % errmsg, atom_node)
+ sub_strings_w = []
+ for index in range(atom_node.num_children()):
+ child = atom_node.get_child(index)
+ try:
+ sub_strings_w.append(parsestring.parsestr(space, encoding,
child.get_value(),
+
unicode_literals))
+ except error.OperationError as e:
+ if not e.match(space, space.w_UnicodeError):
+ raise
+ # UnicodeError in literal: turn into SyntaxError
+ e.normalize_exception(space)
+ errmsg = space.text_w(space.str(e.get_w_value(space)))
+ if child is None:
+ child = atom_node
+ raise self.error('(unicode error) %s' % errmsg, child)
# This implements implicit string concatenation.
if len(sub_strings_w) > 1:
w_sub_strings = space.newlist(sub_strings_w)
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py
b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1250,3 +1250,11 @@
exc = py.test.raises(SyntaxError, self.get_ast, input).value
assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode"
" bytes in position 0-2: truncated \\xXX escape")
+
+ def test_decode_error_in_string_literal_correct_line(self):
+ input = "u'a' u'b'\\\n u'c' u'\\x'"
+ exc = py.test.raises(SyntaxError, self.get_ast, input).value
+ assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode"
+ " bytes in position 0-1: truncated \\xXX escape")
+ assert exc.lineno == 2
+ assert exc.offset == 6
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -210,6 +210,21 @@
def _set_mapdict_storage_and_map(self, storage, map):
raise NotImplementedError
+
+ # -------------------------------------------------------------------
+ # cpyext support
+ # these functions will only be seen by the annotator if we translate
+ # with the cpyext module
+
+ def _cpyext_as_pyobj(self, space):
+ from pypy.module.cpyext.pyobject import w_root_as_pyobj
+ return w_root_as_pyobj(self, space)
+
+ def _cpyext_attach_pyobj(self, space, py_obj):
+ from pypy.module.cpyext.pyobject import w_root_attach_pyobj
+ return w_root_attach_pyobj(self, space, py_obj)
+
+
# -------------------------------------------------------------------
def is_w(self, space, w_other):
diff --git a/pypy/interpreter/test/test_unicodehelper.py
b/pypy/interpreter/test/test_unicodehelper.py
--- a/pypy/interpreter/test/test_unicodehelper.py
+++ b/pypy/interpreter/test/test_unicodehelper.py
@@ -1,5 +1,6 @@
import pytest
import struct
+import sys
from pypy.interpreter.unicodehelper import (
encode_utf8, decode_utf8, unicode_encode_utf_32_be)
@@ -26,7 +27,10 @@
got = decode_utf8(space, "\xed\xa0\x80\xed\xb0\x80")
assert map(ord, got) == [0xd800, 0xdc00]
got = decode_utf8(space, "\xf0\x90\x80\x80")
- assert map(ord, got) == [0x10000]
+ if sys.maxunicode > 65535:
+ assert map(ord, got) == [0x10000]
+ else:
+ assert map(ord, got) == [55296, 56320]
@pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"])
def test_utf32_surrogates(unich):
diff --git a/pypy/module/__builtin__/operation.py
b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -2,11 +2,13 @@
Interp-level implementation of the basic space operations.
"""
+import math
+
from pypy.interpreter import gateway
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
from rpython.rlib.runicode import UNICHR
-from rpython.rlib.rfloat import isfinite, isinf, round_double, round_away
+from rpython.rlib.rfloat import isfinite, round_double, round_away
from rpython.rlib import rfloat
import __builtin__
@@ -151,7 +153,7 @@
else:
# finite x, and ndigits is not unreasonably large
z = round_double(number, ndigits)
- if isinf(z):
+ if math.isinf(z):
raise oefmt(space.w_OverflowError,
"rounded value too large to represent")
return space.newfloat(z)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -58,6 +58,14 @@
}
+class PyPyDateTime(MixedModule):
+ appleveldefs = {}
+ interpleveldefs = {
+ 'dateinterop': 'interp_pypydatetime.W_DateTime_Date',
+ 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time',
+ 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta',
+ }
+
class Module(MixedModule):
""" PyPy specific "magic" functions. A lot of them are experimental and
subject to change, many are internal. """
@@ -79,7 +87,6 @@
'get_hidden_tb' : 'interp_magic.get_hidden_tb',
'lookup_special' : 'interp_magic.lookup_special',
'do_what_I_mean' : 'interp_magic.do_what_I_mean',
- 'validate_fd' : 'interp_magic.validate_fd',
'resizelist_hint' : 'interp_magic.resizelist_hint',
'newlist_hint' : 'interp_magic.newlist_hint',
'add_memory_pressure' : 'interp_magic.add_memory_pressure',
@@ -108,6 +115,7 @@
"thread": ThreadModule,
"intop": IntOpModule,
"os": OsModule,
+ '_pypydatetime': PyPyDateTime,
}
def setup_after_space_initialization(self):
diff --git a/pypy/module/__pypy__/interp_magic.py
b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -107,14 +107,6 @@
raise oefmt(space.w_TypeError, "expecting dict or list or set object")
return space.newtext(name)
-
-@unwrap_spec(fd='c_int')
-def validate_fd(space, fd):
- try:
- rposix.validate_fd(fd)
- except OSError as e:
- raise wrap_oserror(space, e)
-
def get_console_cp(space):
from rpython.rlib import rwin32 # Windows only
return space.newtuple([
@@ -142,7 +134,7 @@
space.newbool(debug))
@unwrap_spec(estimate=int)
-def add_memory_pressure(estimate):
+def add_memory_pressure(space, estimate):
""" Add memory pressure of estimate bytes. Useful when calling a C function
that internally allocates a big chunk of memory. This instructs the GC to
garbage collect sooner than it would otherwise."""
diff --git a/pypy/module/__pypy__/interp_pypydatetime.py
b/pypy/module/__pypy__/interp_pypydatetime.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/interp_pypydatetime.py
@@ -0,0 +1,24 @@
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app
+from rpython.tool.sourcetools import func_with_new_name
+
+def create_class(name):
+ class W_Class(W_Root):
+ 'builtin base clasee for datetime.%s to allow interop with cpyext' %
name
+ def descr_new__(space, w_type):
+ return space.allocate_instance(W_Class, w_type)
+
+ W_Class.typedef = TypeDef(name,
+ __new__ = interp2app(func_with_new_name(
+ W_Class.descr_new__.im_func,
+ '%s_new' % (name,))),
+ )
+ W_Class.typedef.acceptable_as_base_class = True
+ return W_Class
+
+W_DateTime_Time = create_class('pypydatetime_time')
+W_DateTime_Date = create_class('pypydatetime_date')
+W_DateTime_Delta = create_class('pypydatetime_delta')
+
+
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.11.3"
+VERSION = "1.11.4"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/allocator.py
b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -21,13 +21,13 @@
if self.w_alloc is None:
if self.should_clear_after_alloc:
ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
- flavor='raw', zero=True,
- add_memory_pressure=True)
+ flavor='raw', zero=True)
else:
ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
- flavor='raw', zero=False,
- add_memory_pressure=True)
- return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+ flavor='raw', zero=False)
+ w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+ rgc.add_memory_pressure(datasize, w_res)
+ return w_res
else:
w_raw_cdata = space.call_function(self.w_alloc,
space.newint(datasize))
@@ -53,7 +53,7 @@
if self.w_free is not None:
res.w_free = self.w_free
res.register_finalizer(space)
- rgc.add_memory_pressure(datasize)
+ rgc.add_memory_pressure(datasize, res)
return res
@unwrap_spec(w_init=WrappedDefault(None))
diff --git a/pypy/module/_cffi_backend/cdataobj.py
b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -447,7 +447,10 @@
with self as ptr:
w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
if size != 0:
- rgc.add_memory_pressure(size)
+ if isinstance(w_res, W_CDataGCP):
+ rgc.add_memory_pressure(size, w_res)
+ else:
+ rgc.add_memory_pressure(size, self)
return w_res
def unpack(self, length):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.11.3", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.4", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
diff --git a/pypy/module/_codecs/test/test_codecs.py
b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -538,11 +538,16 @@
assert '\x00'.decode('unicode-internal', 'ignore') == ''
def test_backslashreplace(self):
+ import sys
sin = u"a\xac\u1234\u20ac\u8000\U0010ffff"
- expected = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff"
- assert sin.encode('ascii', 'backslashreplace') == expected
- expected = "a\xac\\u1234\xa4\\u8000\\U0010ffff"
- assert sin.encode("iso-8859-15", "backslashreplace") == expected
+ if sys.maxunicode > 65535:
+ expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff"
+ expected_8859 = "a\xac\\u1234\xa4\\u8000\\U0010ffff"
+ else:
+ expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff"
+ expected_8859 = "a\xac\\u1234\xa4\\u8000\\udbff\\udfff"
+ assert sin.encode('ascii', 'backslashreplace') == expected_ascii
+ assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859
def test_badhandler(self):
import codecs
diff --git a/pypy/module/_hashlib/interp_hashlib.py
b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -61,7 +61,8 @@
ctx = ropenssl.EVP_MD_CTX_new()
if ctx is None:
raise MemoryError
- rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
+ rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size,
+ self)
try:
if copy_from:
if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
diff --git a/pypy/module/_io/test/test_interp_textio.py
b/pypy/module/_io/test/test_interp_textio.py
--- a/pypy/module/_io/test/test_interp_textio.py
+++ b/pypy/module/_io/test/test_interp_textio.py
@@ -1,6 +1,6 @@
import pytest
try:
- from hypothesis import given, strategies as st
+ from hypothesis import given, strategies as st, settings
except ImportError:
pytest.skip("hypothesis required")
import os
@@ -29,6 +29,7 @@
@given(data=st_readline(),
mode=st.sampled_from(['\r', '\n', '\r\n', '']))
+@settings(deadline=None)
def test_readline(space, data, mode):
txt, limits = data
w_stream = W_BytesIO(space)
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1316,8 +1316,9 @@
if not ctx:
raise ssl_error(space, "failed to allocate SSL context")
- rgc.add_memory_pressure(10 * 1024 * 1024)
self = space.allocate_instance(_SSLContext, w_subtype)
+ assert isinstance(self, _SSLContext)
+ rgc.add_memory_pressure(10 * 1024 * 1024, self)
self.ctx = ctx
self.check_hostname = False
self.register_finalizer(space)
diff --git a/pypy/module/array/interp_array.py
b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1090,12 +1090,16 @@
start, stop, step, size = self.space.decode_index4(w_idx, self.len)
assert step != 0
if w_item.len != size or self is w_item:
- # XXX this is a giant slow hack
- w_lst = self.descr_tolist(space)
- w_item = space.call_method(w_item, 'tolist')
- space.setitem(w_lst, w_idx, w_item)
- self.setlen(0)
- self.fromsequence(w_lst)
+ if start == self.len and step > 0:
+ # we actually want simply extend()
+ self.extend(w_item)
+ else:
+ # XXX this is a giant slow hack
+ w_lst = self.descr_tolist(space)
+ w_item = space.call_method(w_item, 'tolist')
+ space.setitem(w_lst, w_idx, w_item)
+ self.setlen(0)
+ self.fromsequence(w_lst)
else:
j = 0
buf = self.get_buffer()
diff --git a/pypy/module/array/test/test_array.py
b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -267,6 +267,12 @@
b = self.array('u', u'hi')
assert len(b) == 2 and b[0] == 'h' and b[1] == 'i'
+ def test_setslice_to_extend(self):
+ a = self.array('i')
+ a[0:1] = self.array('i', [9])
+ a[1:5] = self.array('i', [99])
+ assert list(a) == [9, 99]
+
def test_sequence(self):
a = self.array('i', [1, 2, 3, 4])
assert len(a) == 4
diff --git a/pypy/module/cmath/test/test_cmath.py
b/pypy/module/cmath/test/test_cmath.py
--- a/pypy/module/cmath/test/test_cmath.py
+++ b/pypy/module/cmath/test/test_cmath.py
@@ -1,5 +1,4 @@
from __future__ import with_statement
-from rpython.rlib.rfloat import copysign, isnan, isinf
from pypy.module.cmath import interp_cmath
import os, sys, math
@@ -11,7 +10,7 @@
assert isinstance(sqrt_special_values[5][1], tuple)
assert sqrt_special_values[5][1][0] == 1e200 * 1e200
assert sqrt_special_values[5][1][1] == -0.
- assert copysign(1., sqrt_special_values[5][1][1]) == -1.
+ assert math.copysign(1., sqrt_special_values[5][1][1]) == -1.
class AppTestCMath:
@@ -142,12 +141,12 @@
"""
# special values testing
- if isnan(a):
- if isnan(b):
+ if math.isnan(a):
+ if math.isnan(b):
return
raise AssertionError(msg + '%r should be nan' % (b,))
- if isinf(a):
+ if math.isinf(a):
if a == b:
return
raise AssertionError(msg + 'finite result where infinity expected: '
@@ -158,8 +157,7 @@
# and b to have opposite signs; in practice these hardly ever
# occur).
if not a and not b:
- # only check it if we are running on top of CPython >= 2.6
- if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b):
+ if math.copysign(1., a) != math.copysign(1., b):
raise AssertionError(msg + 'zero has wrong sign: expected %r, '
'got %r' % (a, b))
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -30,7 +30,7 @@
from pypy.module.__builtin__.interp_classobj import W_ClassObject
from pypy.module.micronumpy.base import W_NDimArray
from rpython.rlib.entrypoint import entrypoint_lowlevel
-from rpython.rlib.rposix import is_valid_fd, validate_fd
+from rpython.rlib.rposix import FdValidator
from rpython.rlib.unroll import unrolling_iterable
from rpython.rlib.objectmodel import specialize
from pypy.module import exceptions
@@ -96,25 +96,24 @@
dash = ''
def fclose(fp):
- if not is_valid_fd(c_fileno(fp)):
+ try:
+ with FdValidator(c_fileno(fp)):
+ return c_fclose(fp)
+ except IOError:
return -1
- return c_fclose(fp)
def fwrite(buf, sz, n, fp):
- validate_fd(c_fileno(fp))
- return c_fwrite(buf, sz, n, fp)
+ with FdValidator(c_fileno(fp)):
+ return c_fwrite(buf, sz, n, fp)
def fread(buf, sz, n, fp):
- validate_fd(c_fileno(fp))
- return c_fread(buf, sz, n, fp)
+ with FdValidator(c_fileno(fp)):
+ return c_fread(buf, sz, n, fp)
_feof = rffi.llexternal('feof', [FILEP], rffi.INT)
def feof(fp):
- validate_fd(c_fileno(fp))
- return _feof(fp)
-
-def is_valid_fp(fp):
- return is_valid_fd(c_fileno(fp))
+ with FdValidator(c_fileno(fp)):
+ return _feof(fp)
pypy_decl = 'pypy_decl.h'
udir.join(pypy_decl).write("/* Will be filled later */\n")
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -1,43 +1,29 @@
from rpython.rtyper.lltypesystem import rffi, lltype
from rpython.rtyper.annlowlevel import llhelper
-from pypy.module.cpyext.pyobject import PyObject, make_ref
+from rpython.rlib.rarithmetic import widen
+from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr,
+ decref, as_pyobj, incref)
from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct,
- PyObjectFields)
+ PyObjectFields, cts, parse_dir, bootstrap_function, slot_function,
+ Py_TPFLAGS_HEAPTYPE)
from pypy.module.cpyext.import_ import PyImport_Import
from pypy.module.cpyext.typeobject import PyTypeObjectPtr
from pypy.interpreter.error import OperationError
+from pypy.module.__pypy__.interp_pypydatetime import (W_DateTime_Date,
+ W_DateTime_Time, W_DateTime_Delta)
from rpython.tool.sourcetools import func_renamer
-# API import function
+cts.parse_header(parse_dir / 'cpyext_datetime.h')
-PyDateTime_CAPI = cpython_struct(
- 'PyDateTime_CAPI',
- (('DateType', PyTypeObjectPtr),
- ('DateTimeType', PyTypeObjectPtr),
- ('TimeType', PyTypeObjectPtr),
- ('DeltaType', PyTypeObjectPtr),
- ('TZInfoType', PyTypeObjectPtr),
- ('Date_FromDate', lltype.Ptr(lltype.FuncType(
- [rffi.INT_real, rffi.INT_real, rffi.INT_real, PyTypeObjectPtr],
- PyObject))),
- ('Time_FromTime', lltype.Ptr(lltype.FuncType(
- [rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
- PyObject, PyTypeObjectPtr],
- PyObject))),
- ('DateTime_FromDateAndTime', lltype.Ptr(lltype.FuncType(
- [rffi.INT_real, rffi.INT_real, rffi.INT_real,
- rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
- PyObject, PyTypeObjectPtr],
- PyObject))),
- ('Delta_FromDelta', lltype.Ptr(lltype.FuncType(
- [rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
- PyTypeObjectPtr],
- PyObject))),
- ))
+PyDateTime_CAPI = cts.gettype('PyDateTime_CAPI')
+
+datetimeAPI_global = []
@cpython_api([], lltype.Ptr(PyDateTime_CAPI))
def _PyDateTime_Import(space):
+ if len(datetimeAPI_global) >0:
+ return datetimeAPI_global[0]
datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw',
track_allocation=False)
@@ -46,6 +32,10 @@
w_type = space.getattr(w_datetime, space.newtext("date"))
datetimeAPI.c_DateType = rffi.cast(
PyTypeObjectPtr, make_ref(space, w_type))
+ # convenient place to modify this, needed since the make_typedescr attach
+ # links the "wrong" struct to W_DateTime_Date, which in turn is needed
+ # because datetime, with a tzinfo entry, inherits from date, without one
+ datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO)
w_type = space.getattr(w_datetime, space.newtext("datetime"))
datetimeAPI.c_DateTimeType = rffi.cast(
@@ -76,21 +66,13 @@
_PyDelta_FromDelta.api_func.functype,
_PyDelta_FromDelta.api_func.get_wrapper(space))
+ datetimeAPI_global.append(datetimeAPI)
return datetimeAPI
-PyDateTime_DateStruct = lltype.ForwardReference()
-PyDateTime_TimeStruct = lltype.ForwardReference()
-PyDateTime_DateTimeStruct = lltype.ForwardReference()
-cpython_struct("PyDateTime_Date", PyObjectFields, PyDateTime_DateStruct)
-PyDateTime_Date = lltype.Ptr(PyDateTime_DateStruct)
-cpython_struct("PyDateTime_Time", PyObjectFields, PyDateTime_TimeStruct)
-PyDateTime_Time = lltype.Ptr(PyDateTime_TimeStruct)
-cpython_struct("PyDateTime_DateTime", PyObjectFields,
PyDateTime_DateTimeStruct)
-PyDateTime_DateTime = lltype.Ptr(PyDateTime_DateTimeStruct)
-
-PyDeltaObjectStruct = lltype.ForwardReference()
-cpython_struct("PyDateTime_Delta", PyObjectFields, PyDeltaObjectStruct)
-PyDateTime_Delta = lltype.Ptr(PyDeltaObjectStruct)
+PyDateTime_Time = cts.gettype('PyDateTime_Time*')
+PyDateTime_DateTime = cts.gettype('PyDateTime_DateTime*')
+PyDateTime_Date = cts.gettype('PyDateTime_Date*')
+PyDateTime_Delta = cts.gettype('PyDateTime_Delta*')
# Check functions
@@ -129,6 +111,75 @@
PyTZInfo_Check, PyTZInfo_CheckExact = make_check_function(
"PyTZInfo_Check", "tzinfo")
+@bootstrap_function
+def init_datetime(space):
+ # no realize functions since there are no getters
+ make_typedescr(W_DateTime_Time.typedef,
+ basestruct=PyDateTime_Time.TO,
+ attach=type_attach,
+ dealloc=type_dealloc,
+ )
+
+ # why do we need date_dealloc? Since W_DateTime_Date is the base class for
+ # app level datetime.date. If a c-extension class uses datetime.date for
its
+ # base class and defines a tp_dealloc, we will get this:
+ # c_class->tp_dealloc == tp_dealloc_func
+ # c_class->tp_base == datetime.date,
+ # datetime.date->tp_dealloc = _PyPy_subtype_dealloc
+ # datetime.date->tp_base = W_DateTime_Date
+ # W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc
+ # but _PyPy_subtype_dealloc will call tp_dealloc_func, which can call its
+ # base's tp_dealloc and we get recursion. So break the recursion by setting
+ # W_DateTime_Date->tp_dealloc
+ make_typedescr(W_DateTime_Date.typedef,
+ basestruct=PyDateTime_DateTime.TO,
+ attach=type_attach,
+ dealloc=date_dealloc,
+ )
+
+ make_typedescr(W_DateTime_Delta.typedef,
+ basestruct=PyDateTime_Delta.TO,
+ attach=timedeltatype_attach,
+ )
+
+def type_attach(space, py_obj, w_obj, w_userdata=None):
+ '''Fills a newly allocated py_obj from the w_obj
+ '''
+ if space.type(w_obj).name == 'date':
+ # No tzinfo
+ return
+ py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+ w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
+ if space.is_none(w_tzinfo):
+ py_datetime.c_hastzinfo = cts.cast('unsigned char', 0)
+ py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO)
+ else:
+ py_datetime.c_hastzinfo = cts.cast('unsigned char', 1)
+ py_datetime.c_tzinfo = make_ref(space, w_tzinfo)
+
+@slot_function([PyObject], lltype.Void)
+def type_dealloc(space, py_obj):
+ py_datetime = rffi.cast(PyDateTime_Time, py_obj)
+ if (widen(py_datetime.c_hastzinfo) != 0):
+ decref(space, py_datetime.c_tzinfo)
+ from pypy.module.cpyext.object import _dealloc
+ _dealloc(space, py_obj)
+
+@slot_function([PyObject], lltype.Void)
+def date_dealloc(space, py_obj):
+ from pypy.module.cpyext.object import _dealloc
+ _dealloc(space, py_obj)
+
+def timedeltatype_attach(space, py_obj, w_obj, w_userdata=None):
+ "Fills a newly allocated py_obj from the w_obj"
+ py_delta = rffi.cast(PyDateTime_Delta, py_obj)
+ days = space.int_w(space.getattr(w_obj, space.newtext('days')))
+ py_delta.c_days = cts.cast('int', days)
+ seconds = space.int_w(space.getattr(w_obj, space.newtext('seconds')))
+ py_delta.c_seconds = cts.cast('int', seconds)
+ microseconds = space.int_w(space.getattr(w_obj,
space.newtext('microseconds')))
+ py_delta.c_microseconds = cts.cast('int', microseconds)
+
# Constructors. They are better used as macros.
@cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, PyTypeObjectPtr],
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -5,7 +5,7 @@
from rpython.rlib.rarithmetic import widen
from pypy.module.cpyext.api import (
cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
- cpython_struct, is_valid_fp)
+ cpython_struct)
from pypy.module.cpyext.pyobject import PyObject
from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno
from pypy.module.cpyext.funcobject import PyCodeObject
@@ -155,22 +155,19 @@
BUF_SIZE = 8192
source = ""
filename = rffi.charp2str(filename)
- buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw')
- if not is_valid_fp(fp):
- lltype.free(buf, flavor='raw')
- PyErr_SetFromErrno(space, space.w_IOError)
- return None
- try:
+ with rffi.scoped_alloc_buffer(BUF_SIZE) as buf:
while True:
- count = fread(buf, 1, BUF_SIZE, fp)
+ try:
+ count = fread(buf.raw, 1, BUF_SIZE, fp)
+ except OSError:
+ PyErr_SetFromErrno(space, space.w_IOError)
+ return
count = rffi.cast(lltype.Signed, count)
- source += rffi.charpsize2str(buf, count)
+ source += rffi.charpsize2str(buf.raw, count)
if count < BUF_SIZE:
if feof(fp):
break
PyErr_SetFromErrno(space, space.w_IOError)
- finally:
- lltype.free(buf, flavor='raw')
return run_string(space, source, filename, start, w_globals, w_locals)
# Undocumented function!
diff --git a/pypy/module/cpyext/include/datetime.h
b/pypy/module/cpyext/include/datetime.h
--- a/pypy/module/cpyext/include/datetime.h
+++ b/pypy/module/cpyext/include/datetime.h
@@ -4,49 +4,11 @@
extern "C" {
#endif
-/* Define structure for C API. */
-typedef struct {
- /* type objects */
- PyTypeObject *DateType;
- PyTypeObject *DateTimeType;
- PyTypeObject *TimeType;
- PyTypeObject *DeltaType;
- PyTypeObject *TZInfoType;
-
- /* constructors */
- PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*);
- PyObject *(*DateTime_FromDateAndTime)(int, int, int, int, int, int, int,
- PyObject*, PyTypeObject*);
- PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*);
- PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*);
-} PyDateTime_CAPI;
+#include "cpyext_datetime.h"
PyAPI_DATA(PyDateTime_CAPI*) PyDateTimeAPI;
-#define PyDateTime_IMPORT \
- do { \
- if(PyDateTimeAPI==NULL) \
- PyDateTimeAPI = _PyDateTime_Import(); \
- } while (0)
-typedef struct {
- PyObject_HEAD
-} PyDateTime_Delta;
-
-typedef struct {
- PyObject_HEAD
-} PyDateTime_Date;
-
-typedef struct {
- PyObject_HEAD
-} PyDateTime_Time;
-
-typedef struct {
- PyObject_HEAD
-} PyDateTime_DateTime;
-
-typedef struct {
- PyObject_HEAD
-} PyDateTime_TZInfo;
+#define PyDateTime_IMPORT (PyDateTimeAPI = _PyDateTime_Import())
/* Macros for accessing constructors in a simplified fashion. */
#define PyDate_FromDate(year, month, day) \
diff --git a/pypy/module/cpyext/include/longobject.h
b/pypy/module/cpyext/include/longobject.h
--- a/pypy/module/cpyext/include/longobject.h
+++ b/pypy/module/cpyext/include/longobject.h
@@ -18,6 +18,9 @@
PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LONG_SUBCLASS)
#define PyLong_CheckExact(op) ((op)->ob_type == &PyLong_Type)
+#define _PyLong_AsByteArray(v, bytes, n, little_endian, is_signed) \
+ _PyLong_AsByteArrayO((PyObject *)(v), bytes, n, little_endian, is_signed)
+
#ifdef __cplusplus
}
#endif
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -3,9 +3,9 @@
cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t,
CONST_STRING, ADDR, CANNOT_FAIL)
from pypy.objspace.std.longobject import W_LongObject
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
-from rpython.rlib.rbigint import rbigint
+from rpython.rlib.rbigint import rbigint, InvalidSignednessError
PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long")
@@ -238,3 +238,26 @@
byteorder = 'big'
result = rbigint.frombytes(s, byteorder, signed != 0)
return space.newlong_from_rbigint(result)
+
+@cpython_api([PyObject, rffi.UCHARP, rffi.SIZE_T,
+ rffi.INT_real, rffi.INT_real], rffi.INT_real, error=-1)
+def _PyLong_AsByteArrayO(space, w_v, bytes, n, little_endian, is_signed):
+ n = rffi.cast(lltype.Signed, n)
+ little_endian = rffi.cast(lltype.Signed, little_endian)
+ signed = rffi.cast(lltype.Signed, is_signed) != 0
+ byteorder = 'little' if little_endian else 'big'
+ bigint = space.bigint_w(w_v)
+ try:
+ digits = bigint.tobytes(n, byteorder, signed)
+ except InvalidSignednessError: # < 0 but not 'signed'
+ # in this case, CPython raises OverflowError even though the C
+ # comments say it should raise TypeError
+ raise oefmt(space.w_OverflowError,
+ "can't convert negative long to unsigned")
+ except OverflowError:
+ raise oefmt(space.w_OverflowError,
+ "long too big to convert")
+ assert len(digits) == n
+ for i in range(n):
+ bytes[i] = rffi.cast(rffi.UCHAR, digits[i])
+ return 0
diff --git a/pypy/module/cpyext/parse/cpyext_datetime.h
b/pypy/module/cpyext/parse/cpyext_datetime.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_datetime.h
@@ -0,0 +1,52 @@
+/* Define structure for C API. */
+typedef struct {
+ /* type objects */
+ PyTypeObject *DateType;
+ PyTypeObject *DateTimeType;
+ PyTypeObject *TimeType;
+ PyTypeObject *DeltaType;
+ PyTypeObject *TZInfoType;
+
+ /* constructors */
+ PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*);
+ PyObject *(*DateTime_FromDateAndTime)(int, int, int, int, int, int, int,
+ PyObject*, PyTypeObject*);
+ PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*);
+ PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*);
+} PyDateTime_CAPI;
+
+typedef struct
+{
+ PyObject_HEAD
+ int days; /* -MAX_DELTA_DAYS <= days <= MAX_DELTA_DAYS */
+ int seconds; /* 0 <= seconds < 24*3600 is invariant */
+ int microseconds; /* 0 <= microseconds < 1000000 is invariant */
+} PyDateTime_Delta;
+
+/* The datetime and time types have an optional tzinfo member,
+ * PyNone if hastzinfo is false.
+ */
+typedef struct
+{
+ PyObject_HEAD
+ unsigned char hastzinfo;
+ PyObject *tzinfo;
+} PyDateTime_Time;
+
+typedef struct
+{
+ PyObject_HEAD
+ unsigned char hastzinfo;
+ PyObject *tzinfo;
+} PyDateTime_DateTime;
+
+
+typedef struct {
+ PyObject_HEAD
+} PyDateTime_Date;
+
+
+typedef struct {
+ PyObject_HEAD
+} PyDateTime_TZInfo;
+
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -10,6 +10,8 @@
PyVarObject, Py_ssize_t, init_function, cts)
from pypy.module.cpyext.state import State
from pypy.objspace.std.typeobject import W_TypeObject
+from pypy.objspace.std.noneobject import W_NoneObject
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
