Author: Philip Jenvey <pjen...@underboss.org> Branch: stdlib-2.7.11 Changeset: r84162:07673190d34f Date: 2016-05-02 18:23 -0700 http://bitbucket.org/pypy/pypy/changeset/07673190d34f/
Log: merge default diff too long, truncating to 2000 out of 36490 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -20,3 +20,5 @@ 5f8302b8bf9f53056e40426f10c72151564e5b19 release-4.0.1 246c9cf22037b11dc0e8c29ce3f291d3b8c5935a release-5.0 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1 +3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1 +b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -111,23 +111,24 @@ Simon Burton Martin Matusiak Konstantin Lopuhin + Stefano Rivera Wenzhu Man John Witulski Laurence Tratt Ivan Sichmann Freitas Greg Price Dario Bertini - Stefano Rivera Mark Pearse Simon Cross + Edd Barrett Andreas Stührk - Edd Barrett Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov + Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Spenser Bauman + Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -139,7 +140,7 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Tobias Pape + Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -170,9 +171,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas + Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila - Devin Jeanpierre Gabriel Lavoie Olivier Dormond Jared Grubb @@ -183,6 +184,7 @@ Victor Stinner Andrews Medina anatoly techtonik + Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -217,7 +219,6 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan - Mark Young Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -225,7 +226,9 @@ Pieter Zieschang Gabriel Lukas Vacek + Kunal Grover Andrew Dalke + Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -240,7 +243,6 @@ Kristjan Valur Jonsson David Lievens Neil Blakey-Milner - Sergey Matyunin Lutz Paelike Lucio Torre Lars Wassermann @@ -252,9 +254,11 @@ Artur Lisiecki Sergey Kishchenko Ignas Mikalajunas + Alecsandru Patrascu Christoph Gerum Martin Blais Lene Wagner + Catalin Gabriel Manciu Tomo Cocoa Kim Jin Su Toni Mattis @@ -291,6 +295,7 @@ Akira Li Gustavo Niemeyer Stephan Busemann + florinpapa Rafał Gałczyński Matt Bogosian Christian Muirhead @@ -305,6 +310,7 @@ Boglarka Vezer Chris Pressey Buck Golemon + Diana Popa Konrad Delong Dinu Gherman Chris Lambacher diff --git a/lib-python/2.7/distutils/cmd.py b/lib-python/2.7/distutils/cmd.py --- a/lib-python/2.7/distutils/cmd.py +++ b/lib-python/2.7/distutils/cmd.py @@ -298,8 +298,16 @@ src_cmd_obj.ensure_finalized() for (src_option, dst_option) in option_pairs: if getattr(self, dst_option) is None: - setattr(self, dst_option, - getattr(src_cmd_obj, src_option)) + try: + setattr(self, dst_option, + getattr(src_cmd_obj, src_option)) + except AttributeError: + # This was added after problems with setuptools 18.4. + # It seems that setuptools 20.9 fixes the problem. + # But e.g. on Ubuntu 14.04 with /usr/bin/virtualenv + # if I say "virtualenv -p pypy venv-pypy" then it + # just installs setuptools 18.4 from some cache... + pass def get_finalized_command(self, command, create=1): diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt --- a/lib-python/stdlib-upgrade.txt +++ b/lib-python/stdlib-upgrade.txt @@ -5,15 +5,23 @@ overly detailed -1. check out the branch vendor/stdlib +0. make sure your working dir is clean +1. check out the branch vendor/stdlib (for 2.7) or vendor/stdlib-3-* (for py3k) + or create branch vendor/stdlib-3-* 2. upgrade the files there + 2a. remove lib-python/2.7/ or lib-python/3/ + 2b. copy the files from the cpython repo + 2c. hg add lib-python/2.7/ or lib-python/3/ + 2d. hg remove --after + 2e. show copied files in cpython repo by running `hg diff --git -r v<old> -r v<new> Lib | grep '^copy \(from\|to\)'` + 2f. fix copies / renames manually by running `hg copy --after <from> <to>` for each copied file 3. update stdlib-version.txt with the output of hg -id from the cpython repo 4. commit -5. update to default/py3k +5. update to default / py3k 6. create a integration branch for the new stdlib (just hg branch stdlib-$version) -7. merge vendor/stdlib +7. merge vendor/stdlib or vendor/stdlib-3-* 8. commit 10. fix issues 11. commit --close-branch -12. merge to default +12. merge to default / py3k diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py --- a/lib_pypy/_collections.py +++ b/lib_pypy/_collections.py @@ -320,8 +320,7 @@ def __reduce_ex__(self, proto): return type(self), (list(self), self.maxlen) - def __hash__(self): - raise TypeError("deque objects are unhashable") + __hash__ = None def __copy__(self): return self.__class__(self, self.maxlen) diff --git a/lib_pypy/_pypy_wait.py b/lib_pypy/_pypy_wait.py --- a/lib_pypy/_pypy_wait.py +++ b/lib_pypy/_pypy_wait.py @@ -1,51 +1,22 @@ -from resource import _struct_rusage, struct_rusage -from ctypes import CDLL, c_int, POINTER, byref -from ctypes.util import find_library +from resource import ffi, lib, _make_struct_rusage __all__ = ["wait3", "wait4"] -libc = CDLL(find_library("c")) -c_wait3 = libc.wait3 -c_wait3.argtypes = [POINTER(c_int), c_int, POINTER(_struct_rusage)] -c_wait3.restype = c_int - -c_wait4 = libc.wait4 -c_wait4.argtypes = [c_int, POINTER(c_int), c_int, POINTER(_struct_rusage)] -c_wait4.restype = c_int - -def create_struct_rusage(c_struct): - return struct_rusage(( - float(c_struct.ru_utime), - float(c_struct.ru_stime), - c_struct.ru_maxrss, - c_struct.ru_ixrss, - c_struct.ru_idrss, - c_struct.ru_isrss, - c_struct.ru_minflt, - c_struct.ru_majflt, - c_struct.ru_nswap, - c_struct.ru_inblock, - c_struct.ru_oublock, - c_struct.ru_msgsnd, - c_struct.ru_msgrcv, - c_struct.ru_nsignals, - c_struct.ru_nvcsw, - c_struct.ru_nivcsw)) def wait3(options): - status = c_int() - _rusage = _struct_rusage() - pid = c_wait3(byref(status), c_int(options), byref(_rusage)) + status = ffi.new("int *") + ru = ffi.new("struct rusage *") + pid = lib.wait3(status, options, ru) - rusage = create_struct_rusage(_rusage) + rusage = _make_struct_rusage(ru) - return pid, status.value, rusage + return pid, status[0], rusage def wait4(pid, options): - status = c_int() - _rusage = _struct_rusage() - pid = c_wait4(c_int(pid), byref(status), c_int(options), byref(_rusage)) + status = ffi.new("int *") + ru = ffi.new("struct rusage *") + pid = lib.wait4(pid, status, options, ru) - rusage = create_struct_rusage(_rusage) + rusage = _make_struct_rusage(ru) - return pid, status.value, rusage + return pid, status[0], rusage diff --git a/lib_pypy/_resource_build.py b/lib_pypy/_resource_build.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_resource_build.py @@ -0,0 +1,118 @@ +from cffi import FFI + +ffi = FFI() + +# Note: we don't directly expose 'struct timeval' or 'struct rlimit' + + +rlimit_consts = ''' +RLIMIT_CPU +RLIMIT_FSIZE +RLIMIT_DATA +RLIMIT_STACK +RLIMIT_CORE +RLIMIT_NOFILE +RLIMIT_OFILE +RLIMIT_VMEM +RLIMIT_AS +RLIMIT_RSS +RLIMIT_NPROC +RLIMIT_MEMLOCK +RLIMIT_SBSIZE +RLIM_INFINITY +RUSAGE_SELF +RUSAGE_CHILDREN +RUSAGE_BOTH +'''.split() + +rlimit_consts = ['#ifdef %s\n\t{"%s", %s},\n#endif\n' % (s, s, s) + for s in rlimit_consts] + + +ffi.set_source("_resource_cffi", """ +#include <sys/types.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/wait.h> + +static const struct my_rlimit_def { + const char *name; + long long value; +} my_rlimit_consts[] = { +$RLIMIT_CONSTS + { NULL, 0 } +}; + +#define doubletime(TV) ((double)(TV).tv_sec + (TV).tv_usec * 0.000001) + +static double my_utime(struct rusage *input) +{ + return doubletime(input->ru_utime); +} + +static double my_stime(struct rusage *input) +{ + return doubletime(input->ru_stime); +} + +static int my_getrlimit(int resource, long long result[2]) +{ + struct rlimit rl; + if (getrlimit(resource, &rl) == -1) + return -1; + result[0] = rl.rlim_cur; + result[1] = rl.rlim_max; + return 0; +} + +static int my_setrlimit(int resource, long long cur, long long max) +{ + struct rlimit rl; + rl.rlim_cur = cur & RLIM_INFINITY; + rl.rlim_max = max & RLIM_INFINITY; + return setrlimit(resource, &rl); +} + +""".replace('$RLIMIT_CONSTS', ''.join(rlimit_consts))) + + +ffi.cdef(""" + +#define RLIM_NLIMITS ... + +const struct my_rlimit_def { + const char *name; + long long value; +} my_rlimit_consts[]; + +struct rusage { + long ru_maxrss; + long ru_ixrss; + long ru_idrss; + long ru_isrss; + long ru_minflt; + long ru_majflt; + long ru_nswap; + long ru_inblock; + long ru_oublock; + long ru_msgsnd; + long ru_msgrcv; + long ru_nsignals; + long ru_nvcsw; + long ru_nivcsw; + ...; +}; + +static double my_utime(struct rusage *); +static double my_stime(struct rusage *); +void getrusage(int who, struct rusage *result); +int my_getrlimit(int resource, long long result[2]); +int my_setrlimit(int resource, long long cur, long long max); + +int wait3(int *status, int options, struct rusage *rusage); +int wait4(int pid, int *status, int options, struct rusage *rusage); +""") + + +if __name__ == "__main__": + ffi.compile() diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.5.2 +Version: 1.6.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.5.2" -__version_info__ = (1, 5, 2) +__version__ = "1.6.0" +__version_info__ = (1, 6, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.5.2" + "\ncompiled with cffi version: 1.6.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -299,6 +299,23 @@ """ return self._backend.string(cdata, maxlen) + def unpack(self, cdata, length): + """Unpack an array of C data of the given length, + returning a Python string/unicode/list. + + If 'cdata' is a pointer to 'char', returns a byte string. + It does not stop at the first null. This is equivalent to: + ffi.buffer(cdata, length)[:] + + If 'cdata' is a pointer to 'wchar_t', returns a unicode string. + 'length' is measured in wchar_t's; it is not the size in bytes. + + If 'cdata' is a pointer to anything else, returns a list of + 'length' items. This is a faster equivalent to: + [cdata[i] for i in range(length)] + """ + return self._backend.unpack(cdata, length) + def buffer(self, cdata, size=-1): """Return a read-write buffer object that references the raw C data pointed to by the given 'cdata'. The 'cdata' must be a pointer or @@ -721,6 +738,26 @@ raise ValueError("ffi.def_extern() is only available on API-mode FFI " "objects") + def list_types(self): + """Returns the user type names known to this FFI instance. + This returns a tuple containing three lists of names: + (typedef_names, names_of_structs, names_of_unions) + """ + typedefs = [] + structs = [] + unions = [] + for key in self._parser._declarations: + if key.startswith('typedef '): + typedefs.append(key[8:]) + elif key.startswith('struct '): + structs.append(key[7:]) + elif key.startswith('union '): + unions.append(key[6:]) + typedefs.sort() + structs.sort() + unions.sort() + return (typedefs, structs, unions) + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -29,7 +29,8 @@ _r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") _r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") _r_cdecl = re.compile(r"\b__cdecl\b") -_r_extern_python = re.compile(r'\bextern\s*"Python"\s*.') +_r_extern_python = re.compile(r'\bextern\s*"' + r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') _r_star_const_space = re.compile( # matches "* const " r"[*]\s*((const|volatile|restrict)\b\s*)+") @@ -88,6 +89,12 @@ # void __cffi_extern_python_start; # int foo(int); # void __cffi_extern_python_stop; + # + # input: `extern "Python+C" int foo(int);` + # output: + # void __cffi_extern_python_plus_c_start; + # int foo(int); + # void __cffi_extern_python_stop; parts = [] while True: match = _r_extern_python.search(csource) @@ -98,7 +105,10 @@ #print ''.join(parts)+csource #print '=>' parts.append(csource[:match.start()]) - parts.append('void __cffi_extern_python_start; ') + if 'C' in match.group(1): + parts.append('void __cffi_extern_python_plus_c_start; ') + else: + parts.append('void __cffi_extern_python_start; ') if csource[endpos] == '{': # grouping variant closing = csource.find('}', endpos) @@ -302,7 +312,7 @@ break # try: - self._inside_extern_python = False + self._inside_extern_python = '__cffi_extern_python_stop' for decl in iterator: if isinstance(decl, pycparser.c_ast.Decl): self._parse_decl(decl) @@ -376,8 +386,10 @@ tp = self._get_type_pointer(tp, quals) if self._options.get('dllexport'): tag = 'dllexport_python ' - elif self._inside_extern_python: + elif self._inside_extern_python == '__cffi_extern_python_start': tag = 'extern_python ' + elif self._inside_extern_python == '__cffi_extern_python_plus_c_start': + tag = 'extern_python_plus_c ' else: tag = 'function ' self._declare(tag + decl.name, tp) @@ -421,11 +433,9 @@ # hack: `extern "Python"` in the C source is replaced # with "void __cffi_extern_python_start;" and # "void __cffi_extern_python_stop;" - self._inside_extern_python = not self._inside_extern_python - assert self._inside_extern_python == ( - decl.name == '__cffi_extern_python_start') + self._inside_extern_python = decl.name else: - if self._inside_extern_python: + if self._inside_extern_python !='__cffi_extern_python_stop': raise api.CDefError( "cannot declare constants or " "variables with 'extern \"Python\"'") diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -1145,11 +1145,11 @@ def _generate_cpy_extern_python_collecttype(self, tp, name): assert isinstance(tp, model.FunctionPtrType) self._do_collect_type(tp) + _generate_cpy_dllexport_python_collecttype = \ + _generate_cpy_extern_python_plus_c_collecttype = \ + _generate_cpy_extern_python_collecttype - def _generate_cpy_dllexport_python_collecttype(self, tp, name): - self._generate_cpy_extern_python_collecttype(tp, name) - - def _generate_cpy_extern_python_decl(self, tp, name, dllexport=False): + def _extern_python_decl(self, tp, name, tag_and_space): prnt = self._prnt if isinstance(tp.result, model.VoidType): size_of_result = '0' @@ -1184,11 +1184,7 @@ size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % ( tp.result.get_c_name(''), size_of_a, tp.result.get_c_name(''), size_of_a) - if dllexport: - tag = 'CFFI_DLLEXPORT' - else: - tag = 'static' - prnt('%s %s' % (tag, tp.result.get_c_name(name_and_arguments))) + prnt('%s%s' % (tag_and_space, tp.result.get_c_name(name_and_arguments))) prnt('{') prnt(' char a[%s];' % size_of_a) prnt(' char *p = a;') @@ -1206,8 +1202,14 @@ prnt() self._num_externpy += 1 + def _generate_cpy_extern_python_decl(self, tp, name): + self._extern_python_decl(tp, name, 'static ') + def _generate_cpy_dllexport_python_decl(self, tp, name): - self._generate_cpy_extern_python_decl(tp, name, dllexport=True) + self._extern_python_decl(tp, name, 'CFFI_DLLEXPORT ') + + def _generate_cpy_extern_python_plus_c_decl(self, tp, name): + self._extern_python_decl(tp, name, '') def _generate_cpy_extern_python_ctx(self, tp, name): if self.target_is_python: @@ -1220,8 +1222,9 @@ self._lsts["global"].append( GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name)) - def _generate_cpy_dllexport_python_ctx(self, tp, name): - self._generate_cpy_extern_python_ctx(tp, name) + _generate_cpy_dllexport_python_ctx = \ + _generate_cpy_extern_python_plus_c_ctx = \ + _generate_cpy_extern_python_ctx def _string_literal(self, s): def _char_repr(c): @@ -1231,7 +1234,7 @@ if c == '\n': return '\\n' return '\\%03o' % ord(c) lines = [] - for line in s.splitlines(True): + for line in s.splitlines(True) or ['']: lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) return ' \\\n'.join(lines) @@ -1319,7 +1322,9 @@ s = s.encode('ascii') super(NativeIO, self).write(s) -def _make_c_or_py_source(ffi, module_name, preamble, target_file): +def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): + if verbose: + print("generating %s" % (target_file,)) recompiler = Recompiler(ffi, module_name, target_is_python=(preamble is None)) recompiler.collect_type_table() @@ -1331,6 +1336,8 @@ with open(target_file, 'r') as f1: if f1.read(len(output) + 1) != output: raise IOError + if verbose: + print("(already up-to-date)") return False # already up-to-date except IOError: tmp_file = '%s.~%d' % (target_file, os.getpid()) @@ -1343,12 +1350,14 @@ os.rename(tmp_file, target_file) return True -def make_c_source(ffi, module_name, preamble, target_c_file): +def make_c_source(ffi, module_name, preamble, target_c_file, verbose=False): assert preamble is not None - return _make_c_or_py_source(ffi, module_name, preamble, target_c_file) + return _make_c_or_py_source(ffi, module_name, preamble, target_c_file, + verbose) -def make_py_source(ffi, module_name, target_py_file): - return _make_c_or_py_source(ffi, module_name, None, target_py_file) +def make_py_source(ffi, module_name, target_py_file, verbose=False): + return _make_c_or_py_source(ffi, module_name, None, target_py_file, + verbose) def _modname_to_file(outputdir, modname, extension): parts = modname.split('.') @@ -1438,7 +1447,8 @@ target = '*' # ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) - updated = make_c_source(ffi, module_name, preamble, c_file) + updated = make_c_source(ffi, module_name, preamble, c_file, + verbose=compiler_verbose) if call_c_compiler: patchlist = [] cwd = os.getcwd() @@ -1458,7 +1468,8 @@ else: if c_file is None: c_file, _ = _modname_to_file(tmpdir, module_name, '.py') - updated = make_py_source(ffi, module_name, c_file) + updated = make_py_source(ffi, module_name, c_file, + verbose=compiler_verbose) if call_c_compiler: return c_file else: @@ -1484,4 +1495,7 @@ def typeof_disabled(*args, **kwds): raise NotImplementedError ffi._typeof = typeof_disabled + for name in dir(ffi): + if not name.startswith('_') and not hasattr(module.ffi, name): + setattr(ffi, name, NotImplemented) return module.lib diff --git a/lib_pypy/ctypes_config_cache/.empty b/lib_pypy/ctypes_config_cache/.empty new file mode 100644 --- /dev/null +++ b/lib_pypy/ctypes_config_cache/.empty @@ -0,0 +1,1 @@ +dummy file to allow old buildbot configuration to run diff --git a/lib_pypy/ctypes_config_cache/__init__.py b/lib_pypy/ctypes_config_cache/__init__.py deleted file mode 100644 diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ /dev/null @@ -1,21 +0,0 @@ -import sys, os -from ctypes_configure import dumpcache - -def dumpcache2(basename, config): - size = 32 if sys.maxint <= 2**32 else 64 - filename = '_%s_%s_.py' % (basename, size) - dumpcache.dumpcache(__file__, filename, config) - # - filename = os.path.join(os.path.dirname(__file__), - '_%s_cache.py' % (basename,)) - g = open(filename, 'w') - print >> g, '''\ -import sys -_size = 32 if sys.maxint <= 2**32 else 64 -# XXX relative import, should be removed together with -# XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib -_mod = __import__("_%s_%%s_" %% (_size,), - globals(), locals(), ["*"]) -globals().update(_mod.__dict__)\ -''' % (basename,) - g.close() diff --git a/lib_pypy/ctypes_config_cache/locale.ctc.py b/lib_pypy/ctypes_config_cache/locale.ctc.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/locale.ctc.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -'ctypes_configure' source for _locale.py. -Run this to rebuild _locale_cache.py. -""" - -from ctypes_configure.configure import (configure, ExternalCompilationInfo, - ConstantInteger, DefinedConstantInteger, SimpleType, check_eci) -import dumpcache - -# ____________________________________________________________ - -_CONSTANTS = [ - 'LC_CTYPE', - 'LC_TIME', - 'LC_COLLATE', - 'LC_MONETARY', - 'LC_MESSAGES', - 'LC_NUMERIC', - 'LC_ALL', - 'CHAR_MAX', -] - -class LocaleConfigure: - _compilation_info_ = ExternalCompilationInfo(includes=['limits.h', - 'locale.h']) -for key in _CONSTANTS: - setattr(LocaleConfigure, key, DefinedConstantInteger(key)) - -config = configure(LocaleConfigure, noerr=True) -for key, value in config.items(): - if value is None: - del config[key] - _CONSTANTS.remove(key) - -# ____________________________________________________________ - -eci = ExternalCompilationInfo(includes=['locale.h', 'langinfo.h']) -HAS_LANGINFO = check_eci(eci) - -if HAS_LANGINFO: - # list of all possible names - langinfo_names = [ - "RADIXCHAR", "THOUSEP", "CRNCYSTR", - "D_T_FMT", "D_FMT", "T_FMT", "AM_STR", "PM_STR", - "CODESET", "T_FMT_AMPM", "ERA", "ERA_D_FMT", "ERA_D_T_FMT", - "ERA_T_FMT", "ALT_DIGITS", "YESEXPR", "NOEXPR", "_DATE_FMT", - ] - for i in range(1, 8): - langinfo_names.append("DAY_%d" % i) - langinfo_names.append("ABDAY_%d" % i) - for i in range(1, 13): - langinfo_names.append("MON_%d" % i) - langinfo_names.append("ABMON_%d" % i) - - class LanginfoConfigure: - _compilation_info_ = eci - nl_item = SimpleType('nl_item') - for key in langinfo_names: - setattr(LanginfoConfigure, key, DefinedConstantInteger(key)) - - langinfo_config = configure(LanginfoConfigure) - for key, value in langinfo_config.items(): - if value is None: - del langinfo_config[key] - langinfo_names.remove(key) - config.update(langinfo_config) - _CONSTANTS += langinfo_names - -# ____________________________________________________________ - -config['ALL_CONSTANTS'] = tuple(_CONSTANTS) -config['HAS_LANGINFO'] = HAS_LANGINFO -dumpcache.dumpcache2('locale', config) diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py deleted file mode 100755 --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ /dev/null @@ -1,56 +0,0 @@ -#! /usr/bin/env python -# Run this script to rebuild all caches from the *.ctc.py files. - -import os, sys - -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) - -import py - -_dirpath = os.path.dirname(__file__) or os.curdir - -from rpython.tool.ansi_print import AnsiLogger -log = AnsiLogger("ctypes_config_cache") - - -def rebuild_one(name): - filename = os.path.join(_dirpath, name) - d = {'__file__': filename} - path = sys.path[:] - try: - sys.path.insert(0, _dirpath) - execfile(filename, d) - finally: - sys.path[:] = path - -def try_rebuild(): - size = 32 if sys.maxint <= 2**32 else 64 - # remove the files '_*_size_.py' - left = {} - for p in os.listdir(_dirpath): - if p.startswith('_') and (p.endswith('_%s_.py' % size) or - p.endswith('_%s_.pyc' % size)): - os.unlink(os.path.join(_dirpath, p)) - elif p.startswith('_') and (p.endswith('_.py') or - p.endswith('_.pyc')): - for i in range(2, len(p)-4): - left[p[:i]] = True - # remove the files '_*_cache.py' if there is no '_*_*_.py' left around - for p in os.listdir(_dirpath): - if p.startswith('_') and (p.endswith('_cache.py') or - p.endswith('_cache.pyc')): - if p[:-9] not in left: - os.unlink(os.path.join(_dirpath, p)) - # - for p in os.listdir(_dirpath): - if p.endswith('.ctc.py'): - try: - rebuild_one(p) - except Exception, e: - log.ERROR("Running %s:\n %s: %s" % ( - os.path.join(_dirpath, p), - e.__class__.__name__, e)) - - -if __name__ == '__main__': - try_rebuild() diff --git a/lib_pypy/ctypes_config_cache/resource.ctc.py b/lib_pypy/ctypes_config_cache/resource.ctc.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/resource.ctc.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -'ctypes_configure' source for resource.py. -Run this to rebuild _resource_cache.py. -""" - - -from ctypes import sizeof -import dumpcache -from ctypes_configure.configure import (configure, - ExternalCompilationInfo, ConstantInteger, DefinedConstantInteger, - SimpleType) - - -_CONSTANTS = ( - 'RLIM_INFINITY', - 'RLIM_NLIMITS', -) -_OPTIONAL_CONSTANTS = ( - 'RLIMIT_CPU', - 'RLIMIT_FSIZE', - 'RLIMIT_DATA', - 'RLIMIT_STACK', - 'RLIMIT_CORE', - 'RLIMIT_RSS', - 'RLIMIT_NPROC', - 'RLIMIT_NOFILE', - 'RLIMIT_OFILE', - 'RLIMIT_MEMLOCK', - 'RLIMIT_AS', - 'RLIMIT_LOCKS', - 'RLIMIT_SIGPENDING', - 'RLIMIT_MSGQUEUE', - 'RLIMIT_NICE', - 'RLIMIT_RTPRIO', - 'RLIMIT_VMEM', - - 'RUSAGE_BOTH', - 'RUSAGE_SELF', - 'RUSAGE_CHILDREN', -) - -# Setup our configure -class ResourceConfigure: - _compilation_info_ = ExternalCompilationInfo(includes=['sys/resource.h']) - rlim_t = SimpleType('rlim_t') -for key in _CONSTANTS: - setattr(ResourceConfigure, key, ConstantInteger(key)) -for key in _OPTIONAL_CONSTANTS: - setattr(ResourceConfigure, key, DefinedConstantInteger(key)) - -# Configure constants and types -config = configure(ResourceConfigure) -config['rlim_t_max'] = (1<<(sizeof(config['rlim_t']) * 8)) - 1 -optional_constants = [] -for key in _OPTIONAL_CONSTANTS: - if config[key] is not None: - optional_constants.append(key) - else: - del config[key] - -config['ALL_CONSTANTS'] = _CONSTANTS + tuple(optional_constants) -dumpcache.dumpcache2('resource', config) diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py --- a/lib_pypy/pwd.py +++ b/lib_pypy/pwd.py @@ -1,4 +1,4 @@ -# ctypes implementation: Victor Stinner, 2008-05-08 +# indirectly based on ctypes implementation: Victor Stinner, 2008-05-08 """ This module provides access to the Unix password database. It is available on all Unix versions. diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -1,15 +1,8 @@ -import sys -if sys.platform == 'win32': - raise ImportError('resource module not available for win32') +"""http://docs.python.org/library/resource""" -# load the platform-specific cache made by running resource.ctc.py -from ctypes_config_cache._resource_cache import * - -from ctypes_support import standard_c_lib as libc -from ctypes_support import get_errno -from ctypes import Structure, c_int, c_long, byref, POINTER +from _resource_cffi import ffi, lib from errno import EINVAL, EPERM -import _structseq +import _structseq, os try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f @@ -18,106 +11,37 @@ class error(Exception): pass +class struct_rusage: + """struct_rusage: Result from getrusage. -# Read required libc functions -_getrusage = libc.getrusage -_getrlimit = libc.getrlimit -_setrlimit = libc.setrlimit -try: - _getpagesize = libc.getpagesize - _getpagesize.argtypes = () - _getpagesize.restype = c_int -except AttributeError: - from os import sysconf - _getpagesize = None +This object may be accessed either as a tuple of + (utime,stime,maxrss,ixrss,idrss,isrss,minflt,majflt, + nswap,inblock,oublock,msgsnd,msgrcv,nsignals,nvcsw,nivcsw) +or via the attributes ru_utime, ru_stime, ru_maxrss, and so on.""" - -class timeval(Structure): - _fields_ = ( - ("tv_sec", c_long), - ("tv_usec", c_long), - ) - def __str__(self): - return "(%s, %s)" % (self.tv_sec, self.tv_usec) - - def __float__(self): - return self.tv_sec + self.tv_usec/1000000.0 - -class _struct_rusage(Structure): - _fields_ = ( - ("ru_utime", timeval), - ("ru_stime", timeval), - ("ru_maxrss", c_long), - ("ru_ixrss", c_long), - ("ru_idrss", c_long), - ("ru_isrss", c_long), - ("ru_minflt", c_long), - ("ru_majflt", c_long), - ("ru_nswap", c_long), - ("ru_inblock", c_long), - ("ru_oublock", c_long), - ("ru_msgsnd", c_long), - ("ru_msgrcv", c_long), - ("ru_nsignals", c_long), - ("ru_nvcsw", c_long), - ("ru_nivcsw", c_long), - ) - -_getrusage.argtypes = (c_int, POINTER(_struct_rusage)) -_getrusage.restype = c_int - - -class struct_rusage: __metaclass__ = _structseq.structseqtype - ru_utime = _structseq.structseqfield(0) - ru_stime = _structseq.structseqfield(1) - ru_maxrss = _structseq.structseqfield(2) - ru_ixrss = _structseq.structseqfield(3) - ru_idrss = _structseq.structseqfield(4) - ru_isrss = _structseq.structseqfield(5) - ru_minflt = _structseq.structseqfield(6) - ru_majflt = _structseq.structseqfield(7) - ru_nswap = _structseq.structseqfield(8) - ru_inblock = _structseq.structseqfield(9) - ru_oublock = _structseq.structseqfield(10) - ru_msgsnd = _structseq.structseqfield(11) - ru_msgrcv = _structseq.structseqfield(12) - ru_nsignals = _structseq.structseqfield(13) - ru_nvcsw = _structseq.structseqfield(14) - ru_nivcsw = _structseq.structseqfield(15) + ru_utime = _structseq.structseqfield(0, "user time used") + ru_stime = _structseq.structseqfield(1, "system time used") + ru_maxrss = _structseq.structseqfield(2, "max. resident set size") + ru_ixrss = _structseq.structseqfield(3, "shared memory size") + ru_idrss = _structseq.structseqfield(4, "unshared data size") + ru_isrss = _structseq.structseqfield(5, "unshared stack size") + ru_minflt = _structseq.structseqfield(6, "page faults not requiring I/O") + ru_majflt = _structseq.structseqfield(7, "page faults requiring I/O") + ru_nswap = _structseq.structseqfield(8, "number of swap outs") + ru_inblock = _structseq.structseqfield(9, "block input operations") + ru_oublock = _structseq.structseqfield(10, "block output operations") + ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") + ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") + ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") + ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") -@builtinify -def rlimit_check_bounds(rlim_cur, rlim_max): - if rlim_cur > rlim_t_max: - raise ValueError("%d does not fit into rlim_t" % rlim_cur) - if rlim_max > rlim_t_max: - raise ValueError("%d does not fit into rlim_t" % rlim_max) - -class rlimit(Structure): - _fields_ = ( - ("rlim_cur", rlim_t), - ("rlim_max", rlim_t), - ) - -_getrlimit.argtypes = (c_int, POINTER(rlimit)) -_getrlimit.restype = c_int -_setrlimit.argtypes = (c_int, POINTER(rlimit)) -_setrlimit.restype = c_int - - -@builtinify -def getrusage(who): - ru = _struct_rusage() - ret = _getrusage(who, byref(ru)) - if ret == -1: - errno = get_errno() - if errno == EINVAL: - raise ValueError("invalid who parameter") - raise error(errno) +def _make_struct_rusage(ru): return struct_rusage(( - float(ru.ru_utime), - float(ru.ru_stime), + lib.my_utime(ru), + lib.my_stime(ru), ru.ru_maxrss, ru.ru_ixrss, ru.ru_idrss, @@ -135,48 +59,59 @@ )) @builtinify +def getrusage(who): + ru = ffi.new("struct rusage *") + if lib.getrusage(who, ru) == -1: + if ffi.errno == EINVAL: + raise ValueError("invalid who parameter") + raise error(ffi.errno) + return _make_struct_rusage(ru) + +@builtinify def getrlimit(resource): - if not(0 <= resource < RLIM_NLIMITS): + if not (0 <= resource < lib.RLIM_NLIMITS): return ValueError("invalid resource specified") - rlim = rlimit() - ret = _getrlimit(resource, byref(rlim)) - if ret == -1: - errno = get_errno() - raise error(errno) - return (rlim.rlim_cur, rlim.rlim_max) + result = ffi.new("long long[2]") + if lib.my_getrlimit(resource, result) == -1: + raise error(ffi.errno) + return (result[0], result[1]) @builtinify -def setrlimit(resource, rlim): - if not(0 <= resource < RLIM_NLIMITS): +def setrlimit(resource, limits): + if not (0 <= resource < lib.RLIM_NLIMITS): return ValueError("invalid resource specified") - rlimit_check_bounds(*rlim) - rlim = rlimit(rlim[0], rlim[1]) - ret = _setrlimit(resource, byref(rlim)) - if ret == -1: - errno = get_errno() - if errno == EINVAL: - return ValueError("current limit exceeds maximum limit") - elif errno == EPERM: - return ValueError("not allowed to raise maximum limit") + limits = tuple(limits) + if len(limits) != 2: + raise ValueError("expected a tuple of 2 integers") + + if lib.my_setrlimit(resource, limits[0], limits[1]) == -1: + if ffi.errno == EINVAL: + raise ValueError("current limit exceeds maximum limit") + elif ffi.errno == EPERM: + raise ValueError("not allowed to raise maximum limit") else: - raise error(errno) + raise error(ffi.errno) + @builtinify def getpagesize(): - if _getpagesize: - return _getpagesize() - else: - try: - return sysconf("SC_PAGE_SIZE") - except ValueError: - # Irix 5.3 has _SC_PAGESIZE, but not _SC_PAGE_SIZE - return sysconf("SC_PAGESIZE") + return os.sysconf("SC_PAGESIZE") -__all__ = ALL_CONSTANTS + ( - 'error', 'timeval', 'struct_rusage', 'rlimit', - 'getrusage', 'getrlimit', 'setrlimit', 'getpagesize', + +def _setup(): + all_constants = [] + p = lib.my_rlimit_consts + while p.name: + name = ffi.string(p.name) + globals()[name] = int(p.value) + all_constants.append(name) + p += 1 + return all_constants + +__all__ = tuple(_setup()) + ( + 'error', 'getpagesize', 'struct_rusage', + 'getrusage', 'getrlimit', 'setrlimit', ) - -del ALL_CONSTANTS +del _setup diff --git a/lib_pypy/syslog.py b/lib_pypy/syslog.py --- a/lib_pypy/syslog.py +++ b/lib_pypy/syslog.py @@ -51,6 +51,8 @@ # if log is not opened, open it now if not _S_log_open: openlog() + if isinstance(message, unicode): + message = str(message) lib.syslog(priority, "%s", message) @builtinify diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -204,15 +204,6 @@ BoolOption("withstrbuf", "use strings optimized for addition (ver 2)", default=False), - BoolOption("withprebuiltchar", - "use prebuilt single-character string objects", - default=False), - - BoolOption("sharesmallstr", - "always reuse the prebuilt string objects " - "(the empty string and potentially single-char strings)", - default=False), - BoolOption("withspecialisedtuple", "use specialised tuples", default=False), @@ -222,39 +213,14 @@ default=False, requires=[("objspace.honor__builtins__", False)]), - BoolOption("withmapdict", - "make instances really small but slow without the JIT", - default=False, - requires=[("objspace.std.getattributeshortcut", True), - ("objspace.std.withtypeversion", True), - ]), - - BoolOption("withrangelist", - "enable special range list implementation that does not " - "actually create the full list until the resulting " - "list is mutated", - default=False), BoolOption("withliststrategies", "enable optimized ways to store lists of primitives ", default=True), - BoolOption("withtypeversion", - "version type objects when changing them", - cmdline=None, - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), - - BoolOption("withmethodcache", - "try to cache method lookups", - default=False, - requires=[("objspace.std.withtypeversion", True), - ("translation.rweakref", True)]), BoolOption("withmethodcachecounter", "try to cache methods and provide a counter in __pypy__. " "for testing purposes only.", - default=False, - requires=[("objspace.std.withmethodcache", True)]), + default=False), IntOption("methodcachesizeexp", " 2 ** methodcachesizeexp is the size of the of the method cache ", default=11), @@ -265,22 +231,10 @@ BoolOption("optimized_list_getitem", "special case the 'list[integer]' expressions", default=False), - BoolOption("getattributeshortcut", - "track types that override __getattribute__", - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), BoolOption("newshortcut", "cache and shortcut calling __new__ from builtin types", - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), + default=False), - BoolOption("withidentitydict", - "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not", - default=False, - # weakrefs needed, because of get_subclasses() - requires=[("translation.rweakref", True)]), ]), ]) @@ -296,15 +250,10 @@ """ # all the good optimizations for PyPy should be listed here if level in ['2', '3', 'jit']: - config.objspace.std.suggest(withrangelist=True) - config.objspace.std.suggest(withmethodcache=True) - config.objspace.std.suggest(withprebuiltchar=True) config.objspace.std.suggest(intshortcut=True) config.objspace.std.suggest(optimized_list_getitem=True) - config.objspace.std.suggest(getattributeshortcut=True) #config.objspace.std.suggest(newshortcut=True) config.objspace.std.suggest(withspecialisedtuple=True) - config.objspace.std.suggest(withidentitydict=True) #if not IS_64_BITS: # config.objspace.std.suggest(withsmalllong=True) @@ -317,16 +266,13 @@ # memory-saving optimizations if level == 'mem': config.objspace.std.suggest(withprebuiltint=True) - config.objspace.std.suggest(withrangelist=True) - config.objspace.std.suggest(withprebuiltchar=True) - config.objspace.std.suggest(withmapdict=True) + config.objspace.std.suggest(withliststrategies=True) if not IS_64_BITS: config.objspace.std.suggest(withsmalllong=True) # extra optimizations with the JIT if level == 'jit': config.objspace.std.suggest(withcelldict=True) - config.objspace.std.suggest(withmapdict=True) def enable_allworkingmodules(config): diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py --- a/pypy/config/test/test_pypyoption.py +++ b/pypy/config/test/test_pypyoption.py @@ -11,12 +11,6 @@ assert conf.objspace.usemodules.gc - conf.objspace.std.withmapdict = True - assert conf.objspace.std.withtypeversion - conf = get_pypy_config() - conf.objspace.std.withtypeversion = False - py.test.raises(ConfigError, "conf.objspace.std.withmapdict = True") - def test_conflicting_gcrootfinder(): conf = get_pypy_config() conf.translation.gc = "boehm" @@ -47,18 +41,10 @@ def test_set_pypy_opt_level(): conf = get_pypy_config() set_pypy_opt_level(conf, '2') - assert conf.objspace.std.getattributeshortcut + assert conf.objspace.std.intshortcut conf = get_pypy_config() set_pypy_opt_level(conf, '0') - assert not conf.objspace.std.getattributeshortcut - -def test_rweakref_required(): - conf = get_pypy_config() - conf.translation.rweakref = False - set_pypy_opt_level(conf, '3') - - assert not conf.objspace.std.withtypeversion - assert not conf.objspace.std.withmethodcache + assert not conf.objspace.std.intshortcut def test_check_documentation(): def check_file_exists(fn): diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -102,15 +102,15 @@ apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \ - tk-dev + tk-dev libgc-dev For the optional lzma module on PyPy3 you will also need ``liblzma-dev``. On Fedora:: - yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \ - lib-sqlite3-devel ncurses-devel expat-devel openssl-devel - (XXX plus the Febora version of libgdbm-dev and tk-dev) + dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \ + lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \ + gdbm-devel For the optional lzma module on PyPy3 you will also need ``xz-devel``. diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -266,7 +266,13 @@ To raise an application-level exception:: - raise OperationError(space.w_XxxError, space.wrap("message")) + from pypy.interpreter.error import oefmt + + raise oefmt(space.w_XxxError, "message") + + raise oefmt(space.w_XxxError, "file '%s' not found in '%s'", filename, dir) + + raise oefmt(space.w_XxxError, "file descriptor '%d' not open", fd) To catch a specific application-level exception:: diff --git a/pypy/doc/config/objspace.std.getattributeshortcut.txt b/pypy/doc/config/objspace.std.getattributeshortcut.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.getattributeshortcut.txt +++ /dev/null @@ -1,1 +0,0 @@ -Performance only: track types that override __getattribute__. diff --git a/pypy/doc/config/objspace.std.methodcachesizeexp.txt b/pypy/doc/config/objspace.std.methodcachesizeexp.txt --- a/pypy/doc/config/objspace.std.methodcachesizeexp.txt +++ b/pypy/doc/config/objspace.std.methodcachesizeexp.txt @@ -1,1 +1,1 @@ -Set the cache size (number of entries) for :config:`objspace.std.withmethodcache`. +Set the cache size (number of entries) for the method cache. diff --git a/pypy/doc/config/objspace.std.withidentitydict.txt b/pypy/doc/config/objspace.std.withidentitydict.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withidentitydict.txt +++ /dev/null @@ -1,21 +0,0 @@ -============================= -objspace.std.withidentitydict -============================= - -* **name:** withidentitydict - -* **description:** enable a dictionary strategy for "by identity" comparisons - -* **command-line:** --objspace-std-withidentitydict - -* **command-line for negation:** --no-objspace-std-withidentitydict - -* **option type:** boolean option - -* **default:** True - - -Enable a dictionary strategy specialized for instances of classes which -compares "by identity", which is the default unless you override ``__hash__``, -``__eq__`` or ``__cmp__``. This strategy will be used only with new-style -classes. diff --git a/pypy/doc/config/objspace.std.withmapdict.txt b/pypy/doc/config/objspace.std.withmapdict.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withmapdict.txt +++ /dev/null @@ -1,5 +0,0 @@ -Enable the new version of "sharing dictionaries". - -See the section in `Standard Interpreter Optimizations`_ for more details. - -.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html#sharing-dicts diff --git a/pypy/doc/config/objspace.std.withmethodcache.txt b/pypy/doc/config/objspace.std.withmethodcache.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withmethodcache.txt +++ /dev/null @@ -1,2 +0,0 @@ -Enable method caching. See the section "Method Caching" in `Standard -Interpreter Optimizations <../interpreter-optimizations.html#method-caching>`__. diff --git a/pypy/doc/config/objspace.std.withmethodcachecounter.txt b/pypy/doc/config/objspace.std.withmethodcachecounter.txt --- a/pypy/doc/config/objspace.std.withmethodcachecounter.txt +++ b/pypy/doc/config/objspace.std.withmethodcachecounter.txt @@ -1,1 +1,1 @@ -Testing/debug option for :config:`objspace.std.withmethodcache`. +Testing/debug option for the method cache. diff --git a/pypy/doc/config/objspace.std.withprebuiltchar.txt b/pypy/doc/config/objspace.std.withprebuiltchar.txt deleted file mode 100644 diff --git a/pypy/doc/config/objspace.std.withrangelist.txt b/pypy/doc/config/objspace.std.withrangelist.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withrangelist.txt +++ /dev/null @@ -1,11 +0,0 @@ -Enable "range list" objects. They are an additional implementation of the Python -``list`` type, indistinguishable for the normal user. Whenever the ``range`` -builtin is called, an range list is returned. As long as this list is not -mutated (and for example only iterated over), it uses only enough memory to -store the start, stop and step of the range. This makes using ``range`` as -efficient as ``xrange``, as long as the result is only used in a ``for``-loop. - -See the section in `Standard Interpreter Optimizations`_ for more details. - -.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html#range-lists - diff --git a/pypy/doc/config/objspace.std.withtypeversion.txt b/pypy/doc/config/objspace.std.withtypeversion.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.std.withtypeversion.txt +++ /dev/null @@ -1,6 +0,0 @@ -This (mostly internal) option enables "type versions": Every type object gets an -(only internally visible) version that is updated when the type's dict is -changed. This is e.g. used for invalidating caches. It does not make sense to -enable this option alone. - -.. internal diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -81,13 +81,13 @@ Simon Burton Martin Matusiak Konstantin Lopuhin + Stefano Rivera Wenzhu Man John Witulski Laurence Tratt Ivan Sichmann Freitas Greg Price Dario Bertini - Stefano Rivera Mark Pearse Simon Cross Andreas Stührk @@ -95,9 +95,10 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov + Spenser Bauman Jeremy Thurgood Paweł Piotr Przeradowski - Spenser Bauman + Tobias Pape Paul deGrandis Ilya Osadchiy marky1991 @@ -109,7 +110,7 @@ Georg Brandl Bert Freudenberg Stian Andreassen - Tobias Pape + Mark Young Wanja Saatkamp Gerald Klix Mike Blume @@ -140,9 +141,9 @@ Yichao Yu Rocco Moretti Gintautas Miliauskas + Devin Jeanpierre Michael Twomey Lucian Branescu Mihaila - Devin Jeanpierre Gabriel Lavoie Olivier Dormond Jared Grubb @@ -153,6 +154,7 @@ Victor Stinner Andrews Medina anatoly techtonik + Sergey Matyunin Stuart Williams Jasper Schulz Christian Hudon @@ -187,7 +189,6 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan - Mark Young Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -195,7 +196,9 @@ Pieter Zieschang Gabriel Lukas Vacek + Kunal Grover Andrew Dalke + Florin Papa Sylvain Thenault Jakub Stasiak Nathan Taylor @@ -210,7 +213,6 @@ Kristjan Valur Jonsson David Lievens Neil Blakey-Milner - Sergey Matyunin Lutz Paelike Lucio Torre Lars Wassermann @@ -222,9 +224,11 @@ Artur Lisiecki Sergey Kishchenko Ignas Mikalajunas + Alecsandru Patrascu Christoph Gerum Martin Blais Lene Wagner + Catalin Gabriel Manciu Tomo Cocoa Kim Jin Su Toni Mattis @@ -261,6 +265,7 @@ Akira Li Gustavo Niemeyer Stephan Busemann + florinpapa Rafał Gałczyński Matt Bogosian Christian Muirhead @@ -275,6 +280,7 @@ Boglarka Vezer Chris Pressey Buck Golemon + Diana Popa Konrad Delong Dinu Gherman Chris Lambacher diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst --- a/pypy/doc/cppyy.rst +++ b/pypy/doc/cppyy.rst @@ -12,9 +12,9 @@ The work on the cling backend has so far been done only for CPython, but bringing it to PyPy is a lot less work than developing it in the first place. -.. _Reflex: http://root.cern.ch/drupal/content/reflex -.. _CINT: http://root.cern.ch/drupal/content/cint -.. _cling: http://root.cern.ch/drupal/content/cling +.. _Reflex: https://root.cern.ch/how/how-use-reflex +.. _CINT: https://root.cern.ch/introduction-cint +.. _cling: https://root.cern.ch/cling .. _llvm: http://llvm.org/ .. _clang: http://clang.llvm.org/ @@ -283,7 +283,8 @@ core reflection set, but for the moment assume we want to have it in the reflection library that we are building for this example. -The ``genreflex`` script can be steered using a so-called `selection file`_, +The ``genreflex`` script can be steered using a so-called `selection file`_ +(see "Generating Reflex Dictionaries") which is a simple XML file specifying, either explicitly or by using a pattern, which classes, variables, namespaces, etc. to select from the given header file. @@ -305,7 +306,7 @@ <function name="BaseFactory" /> </lcgdict> -.. _selection file: http://root.cern.ch/drupal/content/generating-reflex-dictionaries +.. _selection file: https://root.cern.ch/how/how-use-reflex Now the reflection info can be generated and compiled:: @@ -811,7 +812,7 @@ immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment variable. -.. _PyROOT: http://root.cern.ch/drupal/content/pyroot +.. _PyROOT: https://root.cern.ch/pyroot There are a couple of minor differences between PyCintex and cppyy, most to do with naming. diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -387,6 +387,14 @@ wrappers. On PyPy we can't tell the difference, so ``ismethod([].__add__) == ismethod(list.__add__) == True``. +* in CPython, the built-in types have attributes that can be + implemented in various ways. Depending on the way, if you try to + write to (or delete) a read-only (or undeletable) attribute, you get + either a ``TypeError`` or an ``AttributeError``. PyPy tries to + strike some middle ground between full consistency and full + compatibility here. This means that a few corner cases don't raise + the same exception, like ``del (lambda:None).__closure__``. + * in pure Python, if you write ``class A(object): def f(self): pass`` and have a subclass ``B`` which doesn't override ``f()``, then ``B.f(x)`` still checks that ``x`` is an instance of ``B``. In diff --git a/pypy/doc/dir-reference.rst b/pypy/doc/dir-reference.rst --- a/pypy/doc/dir-reference.rst +++ b/pypy/doc/dir-reference.rst @@ -21,7 +21,7 @@ :source:`pypy/doc/discussion/` drafts of ideas and documentation -:source:`pypy/goal/` our :ref:`main PyPy-translation scripts <translate-pypy>` +:source:`pypy/goal/` our main PyPy-translation scripts live here :source:`pypy/interpreter/` :doc:`bytecode interpreter <interpreter>` and related objects diff --git a/pypy/doc/discussions.rst b/pypy/doc/discussions.rst --- a/pypy/doc/discussions.rst +++ b/pypy/doc/discussions.rst @@ -13,3 +13,4 @@ discussion/improve-rpython discussion/ctypes-implementation discussion/jit-profiler + discussion/rawrefcount diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst --- a/pypy/doc/extending.rst +++ b/pypy/doc/extending.rst @@ -79,7 +79,7 @@ :doc:`Full details <cppyy>` are `available here <cppyy>`. .. _installed separately: http://cern.ch/wlav/reflex-2013-08-14.tar.bz2 -.. _Reflex: http://root.cern.ch/drupal/content/reflex +.. _Reflex: https://root.cern.ch/how/how-use-reflex RPython Mixed Modules diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -106,20 +106,33 @@ For information on which third party extensions work (or do not work) with PyPy see the `compatibility wiki`_. +For more information about how we manage refcounting semamtics see +rawrefcount_ + .. _compatibility wiki: https://bitbucket.org/pypy/compatibility/wiki/Home .. _cffi: http://cffi.readthedocs.org/ +.. _rawrefcount: discussion/rawrefcount.html On which platforms does PyPy run? --------------------------------- -PyPy is regularly and extensively tested on Linux machines. It mostly +PyPy currently supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +PyPy is regularly and extensively tested on Linux machines. It works on Mac and Windows: it is tested there, but most of us are running -Linux so fixes may depend on 3rd-party contributions. PyPy's JIT -works on x86 (32-bit or 64-bit) and on ARM (ARMv6 or ARMv7). -Support for POWER (64-bit) is stalled at the moment. +Linux so fixes may depend on 3rd-party contributions. -To bootstrap from sources, PyPy can use either CPython (2.6 or 2.7) or +To bootstrap from sources, PyPy can use either CPython 2.7 or another (e.g. older) PyPy. Cross-translation is not really supported: e.g. to build a 32-bit PyPy, you need to have a 32-bit environment. Cross-translation is only explicitly supported between a 32-bit Intel diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,8 @@ .. toctree:: + release-5.1.1.rst + release-5.1.0.rst release-5.0.1.rst release-5.0.0.rst release-4.0.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-5.1.0.rst whatsnew-5.0.0.rst whatsnew-4.0.1.rst whatsnew-4.0.0.rst diff --git a/pypy/doc/interpreter-optimizations.rst b/pypy/doc/interpreter-optimizations.rst --- a/pypy/doc/interpreter-optimizations.rst +++ b/pypy/doc/interpreter-optimizations.rst @@ -62,29 +62,37 @@ Dictionary Optimizations ~~~~~~~~~~~~~~~~~~~~~~~~ -Multi-Dicts -+++++++++++ +Dict Strategies +++++++++++++++++ -Multi-dicts are a special implementation of dictionaries. It became clear that -it is very useful to *change* the internal representation of an object during -its lifetime. Multi-dicts are a general way to do that for dictionaries: they -provide generic support for the switching of internal representations for -dicts. +Dict strategies are an implementation approach for dictionaries (and lists) +that make it possible to use a specialized representation of the dictionary's +data, while still being able to switch back to a general representation should +that become necessary later. -If you just enable multi-dicts, special representations for empty dictionaries, -for string-keyed dictionaries. In addition there are more specialized dictionary -implementations for various purposes (see below). +Dict strategies are always enabled, by default there are special strategies for +dicts with just string keys, just unicode keys and just integer keys. If one of +those specialized strategies is used, then dict lookup can use much faster +hashing and comparison for the dict keys. There is of course also a strategy +for general keys. -This is now the default implementation of dictionaries in the Python interpreter. +Identity Dicts ++++++++++++++++ -Sharing Dicts +We also have a strategy specialized for keys that are instances of classes +which compares "by identity", which is the default unless you override +``__hash__``, ``__eq__`` or ``__cmp__``. This strategy will be used only with +new-style classes. + + +Map Dicts +++++++++++++ -Sharing dictionaries are a special representation used together with multidicts. -This dict representation is used only for instance dictionaries and tries to -make instance dictionaries use less memory (in fact, in the ideal case the -memory behaviour should be mostly like that of using __slots__). +Map dictionaries are a special representation used together with dict strategies. +This dict strategy is used only for instance dictionaries and tries to +make instance dictionaries use less memory (in fact, usually memory behaviour +should be mostly like that of using ``__slots__``). The idea is the following: Most instances of the same class have very similar attributes, and are even adding these keys to the dictionary in the same order @@ -95,8 +103,6 @@ dicts: the representation of the instance dict contains only a list of values. -A more advanced version of sharing dicts, called *map dicts,* is available -with the :config:`objspace.std.withmapdict` option. List Optimizations @@ -114,8 +120,8 @@ created. This gives the memory and speed behaviour of ``xrange`` and the generality of use of ``range``, and makes ``xrange`` essentially useless. -You can enable this feature with the :config:`objspace.std.withrangelist` -option. +This feature is enabled by default as part of the +:config:`objspace.std.withliststrategies` option. User Class Optimizations @@ -133,8 +139,7 @@ base classes is changed). On subsequent lookups the cached version can be used, as long as the instance did not shadow any of its classes attributes. -You can enable this feature with the :config:`objspace.std.withmethodcache` -option. +This feature is enabled by default. Interpreter Optimizations diff --git a/pypy/doc/introduction.rst b/pypy/doc/introduction.rst --- a/pypy/doc/introduction.rst +++ b/pypy/doc/introduction.rst @@ -1,16 +1,22 @@ What is PyPy? ============= -In common parlance, PyPy has been used to mean two things. The first is the -:ref:`RPython translation toolchain <rpython:index>`, which is a framework for generating -dynamic programming language implementations. And the second is one -particular implementation that is so generated -- -an implementation of the Python_ programming language written in -Python itself. It is designed to be flexible and easy to experiment with. +Historically, PyPy has been used to mean two things. The first is the +:ref:`RPython translation toolchain <rpython:index>` for generating +interpreters for dynamic programming languages. And the second is one +particular implementation of Python_ produced with it. Because RPython +uses the same syntax as Python, this generated version became known as +Python interpreter written in Python. It is designed to be flexible and +easy to experiment with. -This double usage has proven to be confusing, and we are trying to move -away from using the word PyPy to mean both things. From now on we will -try to use PyPy to only mean the Python implementation, and say the +To make it more clear, we start with source code written in RPython, +apply the RPython translation toolchain, and end up with PyPy as a +binary executable. This executable is the Python interpreter. + +Double usage has proven to be confusing, so we've moved away from using +the word PyPy to mean both toolchain and generated interpreter. Now we +use word PyPy to refer to the Python implementation, and explicitly +mention :ref:`RPython translation toolchain <rpython:index>` when we mean the framework. Some older documents, presentations, papers and videos will still have the old diff --git a/pypy/doc/release-5.1.0.rst b/pypy/doc/release-5.1.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-5.1.0.rst @@ -0,0 +1,160 @@ +======== +PyPy 5.1 +======== + +We have released PyPy 5.1, about a month after PyPy 5.0. + +This release includes more improvement to warmup time and memory +requirements. We have seen about a 20% memory requirement reduction and up to +30% warmup time improvement, more detail in the `blog post`_. + +We also now have `fully support for the IBM s390x`_. Since this support is in +`RPython`_, any dynamic language written using RPython, like PyPy, will +automagically be supported on that architecture. + +We updated cffi_ to 1.6, and continue to improve support for the wider +python ecosystem using the PyPy interpreter. + +You can download the PyPy 5.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html +.. _`numpy`: https://bitbucket.org/pypy/numpy +.. _cffi: https://cffi.readthedocs.org +.. _`fully support for the IBM s390x`: http://morepypy.blogspot.com/2016/04/pypy-enterprise-edition.html +.. _`blog post`: http://morepypy.blogspot.com/2016/04/warmup-improvements-more-efficient.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 5.0 released in March 2015) +========================================================= + +* New features: + + * A new jit backend for the IBM s390x, which was a large effort over the past + few months. + + * Add better support for PyUnicodeObject in the C-API compatibility layer + + * Support GNU/kFreeBSD Debian ports in vmprof + + * Add __pypy__._promote + + * Make attrgetter a single type for CPython compatibility + +* Bug Fixes + + * Catch exceptions raised in an exit function + + * Fix a corner case in the JIT + + * Fix edge cases in the cpyext refcounting-compatible semantics + (more work on cpyext compatibility is coming in the ``cpyext-ext`` + branch, but isn't ready yet) + + * Try harder to not emit NEON instructions on ARM processors without NEON + support + + * Improve the rpython posix module system interaction function calls + + * Detect a missing class function implementation instead of calling a random + function + + * Check that PyTupleObjects do not contain any NULLs at the + point of conversion to W_TupleObjects + + * In ctypes, fix _anonymous_ fields of instances + + * Fix JIT issue with unpack() on a Trace which contains half-written operations + + * Fix sandbox startup (a regression in 5.0) + + * Fix possible segfault for classes with mangled mro or __metaclass__ + + * Fix isinstance(deque(), Hashable) on the pure python deque + + * Fix an issue with forkpty() + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy + +* Numpy_: + + * Implemented numpy.where for a single argument + + * Indexing by a numpy scalar now returns a scalar + + * Fix transpose(arg) when arg is a sequence + + * Refactor include file handling, now all numpy ndarray, ufunc, and umath + functions exported from libpypy.so are declared in pypy_numpy.h, which is + included only when building our fork of numpy + + * Add broadcast + +* Performance improvements: + _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit