Author: Ronny Pfannschmidt <opensou...@ronnypfannschmidt.de> Branch: refine-testrunner Changeset: r73784:27f5610c0e09 Date: 2014-10-05 17:51 +0200 http://bitbucket.org/pypy/pypy/changeset/27f5610c0e09/
Log: merge from default diff too long, truncating to 2000 out of 11590 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -367,3 +367,43 @@ Detailed license information is contained in the NOTICE file in the directory. + +Licenses and Acknowledgements for Incorporated Software +======================================================= + +This section is an incomplete, but growing list of licenses and +acknowledgements for third-party software incorporated in the PyPy +distribution. + +License for 'Tcl/Tk' +-------------------- + +This copy of PyPy contains library code that may, when used, result in +the Tcl/Tk library to be loaded. PyPy also includes code that may be +regarded as being a copy of some parts of the Tcl/Tk header files. +You may see a copy of the License for Tcl/Tk in the file +`lib_pypy/_tkinter/license.terms` included here. + +License for 'bzip2' +------------------- + +This copy of PyPy may be linked (dynamically or statically) with the +bzip2 library. You may see a copy of the License for bzip2/libbzip2 at + + http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html + +License for 'openssl' +--------------------- + +This copy of PyPy may be linked (dynamically or statically) with the +openssl library. You may see a copy of the License for OpenSSL at + + https://www.openssl.org/source/license.html + +License for 'gdbm' +------------------ + +The gdbm module includes code from gdbm.h, which is distributed under +the terms of the GPL license version 2 or any later version. Thus the +gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed +under the terms of the GPL license as well. diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/_pytest/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the pytest lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd <this directory> + hg log . -v | less + +then search for all " _pytest/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -53,16 +53,24 @@ self.config = config self.logfile = logfile # preferably line buffered - def write_log_entry(self, testpath, lettercode, longrepr): - py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) + def write_log_entry(self, testpath, lettercode, longrepr, sections=None): + _safeprint("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): - py.builtin.print_(" %s" % line, file=self.logfile) + _safeprint(" %s" % line, file=self.logfile) + if sections is not None and ( + lettercode in ('E', 'F')): # to limit the size of logs + for title, content in sections: + _safeprint(" ---------- %s ----------" % (title,), + file=self.logfile) + for line in content.splitlines(): + _safeprint(" %s" % line, file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) if testpath is None: testpath = report.fspath - self.write_log_entry(testpath, lettercode, longrepr) + self.write_log_entry(testpath, lettercode, longrepr, + getattr(report, 'sections', None)) def pytest_runtest_logreport(self, report): if report.when != "call" and report.passed: @@ -98,3 +106,8 @@ if path is None: path = "cwd:%s" % py.path.local() self.write_log_entry(path, '!', str(excrepr)) + +def _safeprint(s, file): + if isinstance(s, unicode): + s = s.encode('utf-8') + py.builtin.print_(s, file=file) diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -179,25 +179,27 @@ import sys f = open(TESTFN, "r+b") try: - m = mmap.mmap(f.fileno(), mapsize+1) - except ValueError: - # we do not expect a ValueError on Windows - # CAUTION: This also changes the size of the file on disk, and - # later tests assume that the length hasn't changed. We need to - # repair that. + try: + m = mmap.mmap(f.fileno(), mapsize+1) + except ValueError: + # we do not expect a ValueError on Windows + # CAUTION: This also changes the size of the file on disk, and + # later tests assume that the length hasn't changed. We need to + # repair that. + if sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should work on Windows.") + else: + # we expect a ValueError on Unix, but not on Windows + if not sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should raise ValueError.") + m.close() + finally: + f.close() if sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should work on Windows.") - else: - # we expect a ValueError on Unix, but not on Windows - if not sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should raise ValueError.") - m.close() - f.close() - if sys.platform.startswith('win'): - # Repair damage from the resizing test. - f = open(TESTFN, 'r+b') - f.truncate(mapsize) - f.close() + # Repair damage from the resizing test. + f = open(TESTFN, 'r+b') + f.truncate(mapsize) + f.close() # Opening mmap with access=ACCESS_WRITE f = open(TESTFN, "r+b") diff --git a/lib-python/2.7/test/test_select.py b/lib-python/2.7/test/test_select.py --- a/lib-python/2.7/test/test_select.py +++ b/lib-python/2.7/test/test_select.py @@ -57,7 +57,17 @@ del a[-1] return sys.__stdout__.fileno() a[:] = [F()] * 10 - self.assertEqual(select.select([], a, []), ([], a[:5], [])) + result = select.select([], a, []) + # CPython: 'a' ends up with 5 items, because each fileno() + # removes an item and at the middle the iteration stops. + # PyPy: 'a' ends up empty, because the iteration is done on + # a copy of the original list: fileno() is called 10 times. + if test_support.check_impl_detail(cpython=True): + self.assertEqual(len(result[1]), 5) + self.assertEqual(len(a), 5) + if test_support.check_impl_detail(pypy=True): + self.assertEqual(len(result[1]), 10) + self.assertEqual(len(a), 0) def test_main(): test_support.run_unittest(SelectTestCase) diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -286,6 +286,13 @@ lib = ffi.verify(""" +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #include <ncurses.h> #include <panel.h> #include <term.h> diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1242,7 +1242,7 @@ (other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware times") + raise TypeError("can't compare offset-naive and offset-aware times") myhhmm = self._hour * 60 + self._minute - myoff othhmm = other._hour * 60 + other._minute - otoff return _cmp((myhhmm, self._second, self._microsecond), @@ -1838,7 +1838,7 @@ other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware datetimes") + raise TypeError("can't compare offset-naive and offset-aware datetimes") # XXX What follows could be done more efficiently... diff = self - other # this will take offsets into account if diff.days < 0: @@ -1885,7 +1885,7 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError("cannot mix naive and timezone-aware time") + raise TypeError("can't subtract offset-naive and offset-aware datetimes") return base + timedelta(minutes = otoff-myoff) def __hash__(self): diff --git a/py/README-BEFORE-UPDATING b/py/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/py/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the py lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd <this directory> + hg log . -v | less + +then search for all " py/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/py/_path/local.py b/py/_path/local.py --- a/py/_path/local.py +++ b/py/_path/local.py @@ -750,7 +750,8 @@ mkdtemp = classmethod(mkdtemp) def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, - lock_timeout = 172800): # two days + lock_timeout = 172800, # two days + min_timeout = 300): # five minutes """ return unique directory with a number greater than the current maximum one. The number is assumed to start directly after prefix. if keep is true directories with a number less than (maxnum-keep) @@ -818,6 +819,20 @@ for path in rootdir.listdir(): num = parse_num(path) if num is not None and num <= (maxnum - keep): + if min_timeout: + # NB: doing this is needed to prevent (or reduce + # a lot the chance of) the following situation: + # 'keep+1' processes call make_numbered_dir() at + # the same time, they create dirs, but then the + # last process notices the first dir doesn't have + # (yet) a .lock in it and kills it. + try: + t1 = path.lstat().mtime + t2 = lockfile.lstat().mtime + if abs(t2-t1) < min_timeout: + continue # skip directories too recent + except py.error.Error: + continue # failure to get a time, better skip lf = path.join('.lock') try: t1 = lf.lstat().mtime diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '2.3' +version = '2.4' # The full version, including alpha/beta/rc tags. -release = '2.3.0' +release = '2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -111,6 +111,10 @@ of your choice. Typical example: ``--opt=2`` gives a good (but of course slower) Python interpreter without the JIT. + Consider using PyPy instead of CPython in the above command line, + as it is much faster. (Note that ``rpython`` is a Python 2 program, + not Python 3; you need to run either PyPy 2 or CPython 2.) + .. _`optimization level`: config/opt.html If everything works correctly this will create an executable diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -38,14 +38,16 @@ no JIT: windows, linux, os/x sandbox: linux, os/x +* repackage and upload source tar.bz2 to bitbucket and to cobra, as some packagers + prefer a clearly labeled source package * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page * update pypy.org (under extradoc/pypy.org), rebuild and commit * post announcement on morepypy.blogspot.com -* send announcements to pypy-dev, python-list, +* send announcements to twitter.com, pypy-dev, python-list, python-announce, python-dev ... * add a tag on the pypy/jitviewer repo that corresponds to pypy release * add a tag on the codespeed web site that corresponds to pypy release - +* revise versioning at https://readthedocs.org/projects/pypy diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.4.0.rst release-2.3.1.rst release-2.3.0.rst release-2.2.1.rst diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.3.1`_: the latest official release +* `Release 2.4.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.3.1`: http://pypy.org/download.html +.. _`Release 2.4.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.4.0.rst @@ -0,0 +1,122 @@ +================================================= +PyPy 2.4 - Snow White +================================================= + +We're pleased to announce PyPy 2.4, which contains significant performance +enhancements and bug fixes. + +You can download the PyPy 2.4.0 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! We would like to also point out that in +September, `the Python Software Foundation`_ will `match funds`_ for +any donations up to $10k! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy +.. _`the Python Software Foundation`: https://www.python.org/psf/ +.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +Benchmarks improved after internal enhancements in string and +bytearray handling, and a major rewrite of the GIL handling. This means +that external calls are now a lot faster, especially the CFFI ones. It also +means better performance in a lot of corner cases with handling strings or +bytearrays. The main bugfix is handling of many socket objects in your +program which in the long run used to "leak" memory. + +PyPy now uses Python 2.7.8 standard library. + +We fixed a memory leak in IO in the sandbox_ code + +We welcomed more than 12 new contributors, and conducted two Google +Summer of Code projects, as well as other student projects not +directly related to Summer of Code. + + +Issues reported with our previous release were fixed after reports from users on +our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at +#pypy. Here is a summary of the user-facing changes; +for more information see `whats-new`_: + +* Reduced internal copying of bytearray operations + +* Tweak the internal structure of StringBuilder to speed up large string + handling, which becomes advantageous on large programs at the cost of slightly + slower small *benchmark* type programs. + +* Boost performance of thread-local variables in both unjitted and jitted code, + this mostly affects errno handling on linux, which makes external calls + faster. + +* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted + code run *much* faster + +* Optimize errno handling in linux (x86 and x86-64 only) + +* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy + +* Fix performance regression on ufunc(<scalar>, <scalar>) in numpy + +* Classes in the ast module are now distinct from structures used by + the compiler, which simplifies and speeds up translation of our + source code to the PyPy binary interpreter + +* Upgrade stdlib from 2.7.5 to 2.7.8 + +* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i. + No more missing DLLs + +* Many issues were resolved_ since the 2.3.1 release on June 8 + +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.4.0.html +.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +.. _sandbox: http://doc.pypy.org/en/latest/sandbox.html + +We have further improvements on the way: rpython file handling, +numpy linalg compatibility, as well +as improved GC and many smaller improvements. + +Please try it out and let us know what you think. We especially welcome +success stories, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + diff --git a/pypy/doc/release-2.4.rst b/pypy/doc/release-2.4.rst deleted file mode 100644 --- a/pypy/doc/release-2.4.rst +++ /dev/null @@ -1,107 +0,0 @@ -================================================= -PyPy 2.4 - ???????? -================================================= - -We're pleased to announce PyPy 2.4, a significant milestone on it's own right -and the proud parent of our recent PyPy3 and STM releases. - -This release contains several improvements and bugfixes. - -You can download the PyPy 2.4 release here: - - http://pypy.org/download.html - -We would like to thank our donors for the continued support of the PyPy -project, and for those who donate to our three sub-projects. -We've shown quite a bit of progress -but we're slowly running out of funds. -Please consider donating more, or even better convince your employer to donate, -so we can finish those projects! The three sub-projects are: - -* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version - we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version - -* `STM`_ (software transactional memory): We have release a first working version, and -continue to try out new promising paths of acheiving a fast multithreaded python - -* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ - -.. _`Py3k`: http://pypy.org/py3donate.html -.. _`STM`: http://pypy.org/tmdonate2.html -.. _`NumPy`: http://pypy.org/numpydonate.html -.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy - -What is PyPy? -============= - -PyPy is a very compliant Python interpreter, almost a drop-in replacement for -CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison; -note that cpython's speed has not changed since 2.7.2) -due to its integrated tracing JIT compiler. - -This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, -and OpenBSD, -as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. - -While we support 32 bit python on Windows, work on the native Windows 64 -bit python is still stalling, we would welcome a volunteer -to `handle that`_. - -.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org -.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation - -Highlights -========== - -Benchmarks improved after internal improvements in string and bytearray handling, -and a major rewrite of the GIL handling. Many of these improvements are offshoots -of the STM work. - -We merged in Python's 2.7.8 stdlib in a record time of one week, proving the -maturity of our underlying RPython code base and PyPy interpreter. - -We welcomed more than 12 new contributors, and conducted two Google Summer of Code -projects XXX details? - -Issues reported with our previous release were fixed after reports from users on -our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at -#pypy. Here is a summary of the user-facing changes; -for more information see `whats-new`_: - -* Reduced internal copying of bytearray operations - -* Tweak the internal structure of StringBuilder to speed up large string -handling, which becomes advantageous on large programs at the cost of slightly -slower small *benchmark* type programs. - -* Boost performance of thread-local variables in both unjitted and jitted code - -* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted - code run *much* faster - -* Optimize errno handling in linux - -* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy - -* Fix performance regression on ufunc(<scalar>, <scalar>) in numpy - -* Classes in the ast module are now distinct from structures used by the compiler, - which simplifies and speeds up translation of our source code to the PyPy binary - interpreter - -* Upgrade stdlib from 2.7.5 to 2.7.8 - -* - -* Many issues were resolved_ since the 2.3.1 release on June 8 - -.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html -.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved - -Please try it out and let us know what you think. We especially welcome -success stories, we know you are using PyPy, please tell us about it! - -Cheers - -The PyPy Team - diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.4.0.rst @@ -0,0 +1,66 @@ +======================= +What's new in PyPy 2.4+ +======================= + +.. this is a revision shortly after release-2.3.x +.. startrev: ca9b7cf02cf4 + +.. branch: fix-bytearray-complexity +Bytearray operations no longer copy the bytearray unnecessarily + +Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, +``__setslice__``, and ``__len__`` to RPython + +.. branch: stringbuilder2-perf +Give the StringBuilder a more flexible internal structure, with a +chained list of strings instead of just one string. This make it +more efficient when building large strings, e.g. with cStringIO(). + +Also, use systematically jit.conditional_call() instead of regular +branches. This lets the JIT make more linear code, at the cost of +forcing a bit more data (to be passed as arguments to +conditional_calls). I would expect the net result to be a slight +slow-down on some simple benchmarks and a speed-up on bigger +programs. + +.. branch: ec-threadlocal +Change the executioncontext's lookup to be done by reading a thread- +local variable (which is implemented in C using '__thread' if +possible, and pthread_getspecific() otherwise). On Linux x86 and +x86-64, the JIT backend has a special optimization that lets it emit +directly a single MOV from a %gs- or %fs-based address. It seems +actually to give a good boost in performance. + +.. branch: fast-gil +A faster way to handle the GIL, particularly in JIT code. The GIL is +now a composite of two concepts: a global number (it's just set from +1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there +are threads waiting to acquire the GIL, one of them is actively +checking the global number every 0.1 ms to 1 ms. Overall, JIT loops +full of external function calls now run a bit faster (if no thread was +started yet), or a *lot* faster (if threads were started already). + +.. branch: jit-get-errno +Optimize the errno handling in the JIT, notably around external +function calls. Linux-only. + +.. branch: disable_pythonapi +Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this +incompatibility with cpython. Recast sys.dllhandle to an int. + +.. branch: scalar-operations +Fix performance regression on ufunc(<scalar>, <scalar>) in numpy. + +.. branch: pytest-25 +Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, +respectively. + +.. branch: split-ast-classes +Classes in the ast module are now distinct from structures used by the compiler. + +.. branch: stdlib-2.7.8 +Upgrades from 2.7.6 to 2.7.8 + +.. branch: cpybug-seq-radd-rmul +Fix issue #1861 - cpython compatability madness + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,62 +1,14 @@ + ======================= -What's new in PyPy 2.4+ +What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.3.x -.. startrev: ca9b7cf02cf4 +.. this is a revision shortly after release-2.4.x +.. startrev: 7026746cbb1b -.. branch: fix-bytearray-complexity -Bytearray operations no longer copy the bytearray unnecessarily - -Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, -``__setslice__``, and ``__len__`` to RPython - -.. branch: stringbuilder2-perf -Give the StringBuilder a more flexible internal structure, with a -chained list of strings instead of just one string. This make it -more efficient when building large strings, e.g. with cStringIO(). - -Also, use systematically jit.conditional_call() instead of regular -branches. This lets the JIT make more linear code, at the cost of -forcing a bit more data (to be passed as arguments to -conditional_calls). I would expect the net result to be a slight -slow-down on some simple benchmarks and a speed-up on bigger -programs. - -.. branch: ec-threadlocal -Change the executioncontext's lookup to be done by reading a thread- -local variable (which is implemented in C using '__thread' if -possible, and pthread_getspecific() otherwise). On Linux x86 and -x86-64, the JIT backend has a special optimization that lets it emit -directly a single MOV from a %gs- or %fs-based address. It seems -actually to give a good boost in performance. - -.. branch: fast-gil -A faster way to handle the GIL, particularly in JIT code. The GIL is -now a composite of two concepts: a global number (it's just set from -1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there -are threads waiting to acquire the GIL, one of them is actively -checking the global number every 0.1 ms to 1 ms. Overall, JIT loops -full of external function calls now run a bit faster (if no thread was -started yet), or a *lot* faster (if threads were started already). - -.. branch: jit-get-errno -Optimize the errno handling in the JIT, notably around external -function calls. Linux-only. - -.. branch: disable_pythonapi -Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this -incompatibility with cpython. Recast sys.dllhandle to an int. - -.. branch: scalar-operations -Fix performance regression on ufunc(<scalar>, <scalar>) in numpy. - -.. branch: pytest-25 -Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, -respectively. - -.. branch: split-ast-classes -Classes in the ast module are now distinct from structures used by the compiler. - -.. branch: stdlib-2.7.8 -Upgrades from 2.7.6 to 2.7.8 +.. branch: win32-fixes5 +Fix c code generation for msvc so empty "{ }" are avoided in unions, +Avoid re-opening files created with NamedTemporaryFile, +Allocate by 4-byte chunks in rffi_platform, +Skip testing objdump if it does not exist, +and other small adjustments in own tests diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -85,10 +85,13 @@ Abridged method (for -Ojit builds using Visual Studio 2008) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download the versions of all the external packages -from +Download the versions of all the external packages from +https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip +(for 2.4 release and later) or https://bitbucket.org/pypy/pypy/downloads/local.zip -Then expand it into the base directory (base_dir) and modify your environment to reflect this:: +(for pre-2.4 versions) +Then expand it into the base directory (base_dir) and modify your environment +to reflect this:: set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH% set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE% diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -29,6 +29,17 @@ space.w_None) self.startup_called = False + def _cleanup_(self): + """Called by the annotator on prebuilt Module instances. + We don't have many such modules, but for the ones that + show up, remove their __file__ rather than translate it + statically inside the executable.""" + try: + space = self.space + space.delitem(self.w_dict, space.wrap('__file__')) + except OperationError: + pass + def install(self): """NOT_RPYTHON: installs this module into space.builtin_modules""" w_mod = self.space.wrap(self) diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -38,18 +38,15 @@ def cpython_code_signature(code): "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)." argcount = code.co_argcount + varnames = code.co_varnames assert argcount >= 0 # annotator hint - argnames = list(code.co_varnames[:argcount]) + argnames = list(varnames[:argcount]) if code.co_flags & CO_VARARGS: - varargname = code.co_varnames[argcount] + varargname = varnames[argcount] argcount += 1 else: varargname = None - if code.co_flags & CO_VARKEYWORDS: - kwargname = code.co_varnames[argcount] - argcount += 1 - else: - kwargname = None + kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None return Signature(argnames, varargname, kwargname) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -83,12 +83,6 @@ v = PyString_DecodeEscape(space, substr, 'strict', enc) return space.wrap(v) -def hexbyte(val): - result = "%x" % val - if len(result) == 1: - result = "0" + result - return result - def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** # String is utf8-encoded, but 'unicode_escape' expects @@ -108,15 +102,14 @@ # instead. lis.append("u005c") if ord(s[ps]) & 0x80: # XXX inefficient - w, ps = decode_utf8(space, s, ps, end, "utf-32-be") - rn = len(w) - assert rn % 4 == 0 - for i in range(0, rn, 4): - lis.append('\\U') - lis.append(hexbyte(ord(w[i]))) - lis.append(hexbyte(ord(w[i+1]))) - lis.append(hexbyte(ord(w[i+2]))) - lis.append(hexbyte(ord(w[i+3]))) + w, ps = decode_utf8(space, s, ps, end) + for c in w: + # The equivalent of %08x, which is not supported by RPython. + # 7 zeroes are enough for the unicode range, and the + # result still fits in 32-bit. + hexa = hex(ord(c) + 0x10000000) + lis.append('\\U0') + lis.append(hexa[3:]) # Skip 0x and the leading 1 else: lis.append(s[ps]) ps += 1 @@ -136,7 +129,7 @@ # note that the C code has a label here. # the logic is the same. if recode_encoding and ord(s[ps]) & 0x80: - w, ps = decode_utf8(space, s, ps, end, recode_encoding) + w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding) # Append bytes to output buffer. builder.append(w) else: @@ -222,14 +215,18 @@ ch >= 'A' and ch <= 'F') -def decode_utf8(space, s, ps, end, encoding): +def decode_utf8(space, s, ps, end): assert ps >= 0 pt = ps # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps])) - w_v = unicodehelper.encode(space, w_u, encoding) + u = unicodehelper.decode_utf8(space, s[pt:ps]) + return u, ps + +def decode_utf8_recode(space, s, ps, end, recode_encoding): + u, ps = decode_utf8(space, s, ps, end) + w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding) v = space.str_w(w_v) return v, ps diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py --- a/pypy/interpreter/pyparser/test/test_parsestring.py +++ b/pypy/interpreter/pyparser/test/test_parsestring.py @@ -73,11 +73,11 @@ def test_simple_enc_roundtrip(self): space = self.space - s = "'\x81'" + s = "'\x81\\t'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\n'\x81\\t'") def test_multiline_unicode_strings_with_backslash(self): space = self.space diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -945,7 +945,7 @@ prefix = udir.join('pathtest').ensure(dir=1) fake_exe = 'bin/pypy-c' if sys.platform == 'win32': - fake_exe += '.exe' + fake_exe = 'pypy-c.exe' fake_exe = prefix.join(fake_exe).ensure(file=1) expected_path = [str(prefix.join(subdir).ensure(dir=1)) for subdir in ('lib_pypy', @@ -985,6 +985,13 @@ assert sys.path == old_sys_path + [self.goal_dir] app_main.setup_bootstrap_path(self.fake_exe) + if not sys.platform == 'win32': + # an existing file is always 'executable' on windows + assert sys.executable == '' # not executable! + assert sys.path == old_sys_path + [self.goal_dir] + + os.chmod(self.fake_exe, 0755) + app_main.setup_bootstrap_path(self.fake_exe) assert sys.executable == self.fake_exe assert self.goal_dir not in sys.path diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py --- a/pypy/interpreter/test/test_module.py +++ b/pypy/interpreter/test/test_module.py @@ -1,4 +1,5 @@ - +import py +from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module class TestModule: @@ -17,6 +18,18 @@ space.raises_w(space.w_AttributeError, space.delattr, w_m, w('x')) + def test___file__(self, space): + w = space.wrap + m = Module(space, space.wrap('m')) + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + space.setattr(w(m), w('__file__'), w('m.py')) + space.getattr(w(m), w('__file__')) # does not raise + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + + class AppTest_ModuleObject: def test_attr(self): m = __import__('__builtin__') diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -5,6 +5,7 @@ @specialize.memo() def decode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): raise OperationError(space.w_UnicodeDecodeError, @@ -17,6 +18,7 @@ @specialize.memo() def encode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise OperationError(space.w_UnicodeEncodeError, diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -8,6 +8,7 @@ from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P, FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG) from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt @@ -160,6 +161,7 @@ raw_cdata = rffi.cast(rffi.CCHARPP, data)[0] lltype.free(raw_cdata, flavor='raw') lltype.free(buffer, flavor='raw') + keepalive_until_here(args_w) return w_res def get_mustfree_flag(data): diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py --- a/pypy/module/_io/interp_stringio.py +++ b/pypy/module/_io/interp_stringio.py @@ -86,7 +86,7 @@ initval = space.unicode_w(w_initval) size = len(initval) self.resize_buffer(size) - self.buf = [c for c in initval] + self.buf = list(initval) pos = space.getindex_w(w_pos, space.w_TypeError) if pos < 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py --- a/pypy/module/_pypyjson/interp_encoder.py +++ b/pypy/module/_pypyjson/interp_encoder.py @@ -37,16 +37,14 @@ sb = StringBuilder(len(u)) sb.append_slice(s, 0, first) else: + # We used to check if 'u' contains only safe characters, and return + # 'w_string' directly. But this requires an extra pass over all + # characters, and the expected use case of this function, from + # json.encoder, will anyway re-encode a unicode result back to + # a string (with the ascii encoding). This requires two passes + # over the characters. So we may as well directly turn it into a + # string here --- only one pass. u = space.unicode_w(w_string) - for i in range(len(u)): - c = u[i] - if c >= u' ' and c <= u'~' and c != u'"' and c != u'\\': - pass - else: - break - else: - # the input is a unicode with only non-special ascii chars - return w_string sb = StringBuilder(len(u)) first = 0 diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -192,14 +192,14 @@ def test_raw_encode_basestring_ascii(self): import _pypyjson - def check(s, expected_type=str): + def check(s): s = _pypyjson.raw_encode_basestring_ascii(s) - assert type(s) is expected_type + assert type(s) is str return s assert check("") == "" - assert check(u"", expected_type=unicode) == u"" + assert check(u"") == "" assert check("abc ") == "abc " - assert check(u"abc ", expected_type=unicode) == u"abc " + assert check(u"abc ") == "abc " raises(UnicodeDecodeError, check, "\xc0") assert check("\xc2\x84") == "\\u0084" assert check("\xf0\x92\x8d\x85") == "\\ud808\\udf45" diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -759,17 +759,25 @@ # socket's timeout is in seconds, poll's timeout in ms timeout = int(sock_timeout * 1000 + 0.5) - ready = rpoll.poll(fddict, timeout) + try: + ready = rpoll.poll(fddict, timeout) + except rpoll.PollError, e: + message = e.get_msg() + raise ssl_error(space, message, e.errno) else: if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE: return SOCKET_TOO_LARGE_FOR_SELECT - if writing: - r, w, e = rpoll.select([], [sock_fd], [], sock_timeout) - ready = w - else: - r, w, e = rpoll.select([sock_fd], [], [], sock_timeout) - ready = r + try: + if writing: + r, w, e = rpoll.select([], [sock_fd], [], sock_timeout) + ready = w + else: + r, w, e = rpoll.select([sock_fd], [], [], sock_timeout) + ready = r + except rpoll.SelectError as e: + message = e.get_msg() + raise ssl_error(space, message, e.errno) if ready: return SOCKET_OPERATION_OK else: diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -266,10 +266,16 @@ buf = None if typ == rwinreg.REG_DWORD: - if space.isinstance_w(w_value, space.w_int): + if space.is_none(w_value) or ( + space.isinstance_w(w_value, space.w_int) or + space.isinstance_w(w_value, space.w_long)): + if space.is_none(w_value): + value = r_uint(0) + else: + value = space.c_uint_w(w_value) buflen = rffi.sizeof(rwin32.DWORD) buf1 = lltype.malloc(rffi.CArray(rwin32.DWORD), 1, flavor='raw') - buf1[0] = space.uint_w(w_value) + buf1[0] = value buf = rffi.cast(rffi.CCHARP, buf1) elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ: diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -40,7 +40,7 @@ cls.w_tmpfilename = space.wrap(str(udir.join('winreg-temp'))) test_data = [ - ("Int Value", 45, _winreg.REG_DWORD), + ("Int Value", 0xFEDCBA98, _winreg.REG_DWORD), ("Str Value", "A string Value", _winreg.REG_SZ), ("Unicode Value", u"A unicode Value", _winreg.REG_SZ), ("Str Expand", "The path is %path%", _winreg.REG_EXPAND_SZ), @@ -137,9 +137,11 @@ assert 0, "Did not raise" def test_SetValueEx(self): - from _winreg import CreateKey, SetValueEx, REG_BINARY + from _winreg import CreateKey, SetValueEx, REG_BINARY, REG_DWORD key = CreateKey(self.root_key, self.test_key_name) sub_key = CreateKey(key, "sub_key") + SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, None) + SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, 45) for name, value, type in self.test_data: SetValueEx(sub_key, name, 0, type, value) exc = raises(TypeError, SetValueEx, sub_key, 'test_name', None, diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.4.0-alpha0" +#define PYPY_VERSION "2.5.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -407,8 +407,19 @@ -------- numpy.swapaxes : equivalent function """ - if self.is_scalar(): + if axis1 == axis2: return self + n = len(self.get_shape()) + if n <= 1: + return self + if axis1 < 0: + axis1 += n + if axis2 < 0: + axis2 += n + if axis1 < 0 or axis1 >= n: + raise oefmt(space.w_ValueError, "bad axis1 argument to swapaxes") + if axis2 < 0 or axis2 >= n: + raise oefmt(space.w_ValueError, "bad axis2 argument to swapaxes") return self.implementation.swapaxes(space, self, axis1, axis2) def descr_nonzero(self, space): diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2020,6 +2020,14 @@ def test_swapaxes(self): from numpypy import array + x = array([]) + assert x.swapaxes(0, 2) is x + x = array([[1, 2]]) + assert x.swapaxes(0, 0) is x + exc = raises(ValueError, x.swapaxes, -3, 0) + assert exc.value.message == "bad axis1 argument to swapaxes" + exc = raises(ValueError, x.swapaxes, 0, 3) + assert exc.value.message == "bad axis2 argument to swapaxes" # testcases from numpy docstring x = array([[1, 2, 3]]) assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -617,7 +617,7 @@ 'raw_store': 1, 'same_as': 2, 'setarrayitem_gc': 8, - 'setfield_gc': 21, + 'setfield_gc': 22, }) def define_argsort(): diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py --- a/pypy/module/operator/__init__.py +++ b/pypy/module/operator/__init__.py @@ -39,7 +39,7 @@ 'irshift', 'isub', 'itruediv', 'ixor', '_length_hint'] interpleveldefs = { - '_compare_digest': 'interp_operator.compare_digest', + '_compare_digest': 'tscmp.compare_digest', } for name in interp_names: diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py --- a/pypy/module/operator/app_operator.py +++ b/pypy/module/operator/app_operator.py @@ -4,7 +4,7 @@ This module exports a set of operators as functions. E.g. operator.add(x,y) is equivalent to x+y. ''' -from __pypy__ import builtinify + import types @@ -27,7 +27,7 @@ 'getslice(a, b, c) -- Same as a[b:c].' if not isinstance(start, int) or not isinstance(end, int): raise TypeError("an integer is expected") - return a[start:end] + return a[start:end] __getslice__ = getslice def indexOf(a, b): @@ -37,7 +37,7 @@ if x == b: return index index += 1 - raise ValueError, 'sequence.index(x): x not in sequence' + raise ValueError('sequence.index(x): x not in sequence') def isMappingType(obj,): 'isMappingType(a) -- Return True if a has a mapping type, False otherwise.' @@ -58,9 +58,9 @@ def repeat(obj, num): 'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.' if not isinstance(num, (int, long)): - raise TypeError, 'an integer is required' + raise TypeError('an integer is required') if not isSequenceType(obj): - raise TypeError, "non-sequence object can't be repeated" + raise TypeError("non-sequence object can't be repeated") return obj * num @@ -68,59 +68,85 @@ def setslice(a, b, c, d): 'setslice(a, b, c, d) -- Same as a[b:c] = d.' - a[b:c] = d + a[b:c] = d __setslice__ = setslice +def _resolve_attr_chain(chain, obj, idx=0): + obj = getattr(obj, chain[idx]) + if idx + 1 == len(chain): + return obj + else: + return _resolve_attr_chain(chain, obj, idx + 1) + + +class _simple_attrgetter(object): + def __init__(self, attr): + self._attr = attr + + def __call__(self, obj): + return getattr(obj, self._attr) + + +class _single_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return _resolve_attr_chain(self._attrs, obj) + + +class _multi_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return tuple([ + _resolve_attr_chain(attrs, obj) + for attrs in self._attrs + ]) + + def attrgetter(attr, *attrs): + if ( + not isinstance(attr, basestring) or + not all(isinstance(a, basestring) for a in attrs) + ): + def _raise_typeerror(obj): + raise TypeError( + "argument must be a string, not %r" % type(attr).__name__ + ) + return _raise_typeerror if attrs: - getters = [single_attr_getter(a) for a in (attr,) + attrs] - def getter(obj): - return tuple([getter(obj) for getter in getters]) + return _multi_attrgetter([ + a.split(".") for a in [attr] + list(attrs) + ]) + elif "." not in attr: + return _simple_attrgetter(attr) else: - getter = single_attr_getter(attr) - return builtinify(getter) + return _single_attrgetter(attr.split(".")) -def single_attr_getter(attr): - if not isinstance(attr, str): - if not isinstance(attr, unicode): - def _raise_typeerror(obj): - raise TypeError("argument must be a string, not %r" % - (type(attr).__name__,)) - return _raise_typeerror - attr = attr.encode('ascii') - # - def make_getter(name, prevfn=None): - if prevfn is None: - def getter(obj): - return getattr(obj, name) + +class itemgetter(object): + def __init__(self, item, *items): + self._single = not bool(items) + if self._single: + self._idx = item else: - def getter(obj): - return getattr(prevfn(obj), name) - return getter - # - last = 0 - getter = None - while True: - dot = attr.find(".", last) - if dot < 0: break - getter = make_getter(attr[last:dot], getter) - last = dot + 1 - return make_getter(attr[last:], getter) + self._idx = [item] + list(items) + def __call__(self, obj): + if self._single: + return obj[self._idx] + else: + return tuple([obj[i] for i in self._idx]) -def itemgetter(item, *items): - if items: - list_of_indices = [item] + list(items) - def getter(obj): - return tuple([obj[i] for i in list_of_indices]) - else: - def getter(obj): - return obj[item] - return builtinify(getter) +class methodcaller(object): + def __init__(self, method_name, *args, **kwargs): + self._method_name = method_name + self._args = args + self._kwargs = kwargs -def methodcaller(method_name, *args, **kwargs): - def call(obj): - return getattr(obj, method_name)(*args, **kwargs) - return builtinify(call) + def __call__(self, obj): + return getattr(obj, self._method_name)(*self._args, **self._kwargs) diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py --- a/pypy/module/operator/interp_operator.py +++ b/pypy/module/operator/interp_operator.py @@ -1,6 +1,4 @@ -from rpython.rlib.objectmodel import specialize - -from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec @@ -249,33 +247,3 @@ @unwrap_spec(default=int) def _length_hint(space, w_iterable, default): return space.wrap(space.length_hint(w_iterable, default)) - -def compare_digest(space, w_a, w_b): - if ( - space.isinstance_w(w_a, space.w_unicode) and - space.isinstance_w(w_b, space.w_unicode) - ): - return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b))) - if ( - space.isinstance_w(w_a, space.w_unicode) or - space.isinstance_w(w_b, space.w_unicode) - ): - raise oefmt( - space.w_TypeError, - "unsupported operand types(s) or combination of types: '%N' and '%N'", - w_a, - w_b, - ) - else: - return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b))) - - -@specialize.argtype(0, 1) -def tscmp(a, b): - len_a = len(a) - len_b = len(b) - length = min(len(a), len(b)) - res = len_a ^ len_b - for i in xrange(length): - res |= ord(a[i]) ^ ord(b[i]) - return res == 0 diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py --- a/pypy/module/operator/test/test_operator.py +++ b/pypy/module/operator/test/test_operator.py @@ -334,3 +334,9 @@ assert operator._compare_digest(a, b) a, b = mybytes(b"foobar"), mybytes(b"foobaz") assert not operator._compare_digest(a, b) + + def test_compare_digest_unicode(self): + import operator + assert operator._compare_digest(u'asd', u'asd') + assert not operator._compare_digest(u'asd', u'qwe') + raises(TypeError, operator._compare_digest, u'asd', b'qwe') diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py new file mode 100644 --- /dev/null +++ b/pypy/module/operator/test/test_tscmp.py @@ -0,0 +1,28 @@ +from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide + +class TestTimingSafeCompare: + tostr = str + tscmp = staticmethod(pypy_tscmp) + + def test_tscmp_neq(self): + assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3) + + def test_tscmp_eq(self): + assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3) + + def test_tscmp_len(self): + assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3) + + def test_tscmp_nlen(self): + assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3) + + +class TestTimingSafeCompareWide(TestTimingSafeCompare): + tostr = unicode + tscmp = staticmethod(pypy_tscmp_wide) + + def test_tscmp_wide_nonascii(self): + a, b = u"\ud808\udf45", u"\ud808\udf45" + assert self.tscmp(a, b, len(a), len(b)) + a, b = u"\ud808\udf45", u"\ud808\udf45 " + assert not self.tscmp(a, b, len(a), len(b)) diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.c @@ -0,0 +1,80 @@ +/* Derived from CPython 3.3.5's operator.c::_tscmp + */ + +#include <stdlib.h> +#include <wchar.h> +#include "tscmp.h" + +int +pypy_tscmp(const char *a, const char *b, long len_a, long len_b) +{ + /* The volatile type declarations make sure that the compiler has no + * chance to optimize and fold the code in any way that may change + * the timing. + */ + volatile long length; + volatile const char *left; + volatile const char *right; + long i; + char result; + + /* loop count depends on length of b */ + length = len_b; + left = NULL; + right = b; + + /* don't use else here to keep the amount of CPU instructions constant, + * volatile forces re-evaluation + * */ + if (len_a == length) { + left = *((volatile const char**)&a); + result = 0; + } + if (len_a != length) { + left = b; + result = 1; + } + + for (i=0; i < length; i++) { + result |= *left++ ^ *right++; + } + + return (result == 0); +} + +int +pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b) +{ + /* The volatile type declarations make sure that the compiler has no + * chance to optimize and fold the code in any way that may change + * the timing. + */ + volatile long length; + volatile const wchar_t *left; + volatile const wchar_t *right; + long i; + wchar_t result; + + /* loop count depends on length of b */ + length = len_b; + left = NULL; + right = b; + + /* don't use else here to keep the amount of CPU instructions constant, + * volatile forces re-evaluation + * */ + if (len_a == length) { + left = *((volatile const wchar_t**)&a); + result = 0; + } + if (len_a != length) { + left = b; + result = 1; + } + + for (i=0; i < length; i++) { + result |= *left++ ^ *right++; + } + + return (result == 0); +} diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.h @@ -0,0 +1,2 @@ +int pypy_tscmp(const char *, const char *, long, long); +int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long); diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.py @@ -0,0 +1,73 @@ +""" +Provides _compare_digest method, which is a safe comparing to prevent timing +attacks for the hmac module. +""" +import py + +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +from pypy.interpreter.error import oefmt + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('tscmp.h')], + include_dirs=[str(cwd)], + separate_module_files=[cwd.join('tscmp.c')], + export_symbols=['pypy_tscmp', 'pypy_tscmp_wide']) + + +def llexternal(*args, **kwargs): + kwargs.setdefault('compilation_info', eci) + kwargs.setdefault('sandboxsafe', True) + return rffi.llexternal(*args, **kwargs) + + +pypy_tscmp = llexternal( + 'pypy_tscmp', + [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG], + rffi.INT) +pypy_tscmp_wide = llexternal( + 'pypy_tscmp_wide', + [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG], + rffi.INT) + + +def compare_digest(space, w_a, w_b): + """compare_digest(a, b) -> bool + + Return 'a == b'. This function uses an approach designed to prevent + timing analysis, making it appropriate for cryptography. a and b + must both be of the same type: either str (ASCII only), or any type + that supports the buffer protocol (e.g. bytes). + + Note: If a and b are of different lengths, or if an error occurs, a + timing attack could theoretically reveal information about the types + and lengths of a and b--but not their values. + """ + if (space.isinstance_w(w_a, space.w_unicode) and + space.isinstance_w(w_b, space.w_unicode)): + a = space.unicode_w(w_a) + b = space.unicode_w(w_b) + with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf: + with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf: + result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b)) + return space.wrap(rffi.cast(lltype.Bool, result)) + return compare_digest_buffer(space, w_a, w_b) + + +def compare_digest_buffer(space, w_a, w_b): + try: + a_buf = w_a.buffer_w(space, space.BUF_SIMPLE) + b_buf = w_b.buffer_w(space, space.BUF_SIMPLE) + except TypeError: + raise oefmt(space.w_TypeError, + "unsupported operand types(s) or combination of types: " + "'%T' and '%T'", w_a, w_b) + + a = a_buf.as_str() + b = b_buf.as_str() + with rffi.scoped_nonmovingbuffer(a) as a_buf: + with rffi.scoped_nonmovingbuffer(b) as b_buf: + result = pypy_tscmp(a_buf, b_buf, len(a), len(b)) + return space.wrap(rffi.cast(lltype.Bool, result)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -17,13 +17,18 @@ # now we can inline it as call assembler i = 0 j = 0 - while i < 20: + while i < 25: i += 1 j += rec(100) # ID: call_rec return j # - log = self.run(fn, [], threshold=18) - loop, = log.loops_by_filename(self.filepath) + # NB. the parameters below are a bit ad-hoc. After 16 iterations, + # the we trace from the "while" and reach a "trace too long". Then + # in the next execution, we trace the "rec" function from start; + # that's "functrace" below. Then after one or two extra iterations + # we try again from "while", and this time we succeed. + log = self.run(fn, [], threshold=20) + functrace, loop = log.loops_by_filename(self.filepath) assert loop.match_by_id('call_rec', """ ... p53 = call_assembler(..., descr=...) @@ -377,12 +382,16 @@ ... p20 = force_token() p22 = new_with_vtable(...) - p24 = new_array(1, descr=<ArrayP .>) + p24 = new_array_clear(1, descr=<ArrayP .>) p26 = new_with_vtable(ConstClass(W_ListObject)) {{{ setfield_gc(p0, p20, descr=<FieldP .*PyFrame.vable_token .*>) + setfield_gc(p22, ConstPtr(null), descr=<FieldP pypy.interpreter.argument.Arguments.inst_keywords_w .*>) + setfield_gc(p22, ConstPtr(null), descr=<FieldP pypy.interpreter.argument.Arguments.inst_keywords .*>) setfield_gc(p22, 1, descr=<FieldU pypy.interpreter.argument.Arguments.inst__jit_few_keywords .*>) + setfield_gc(p22, ConstPtr(null), descr=<FieldP pypy.interpreter.argument.Arguments.inst_keyword_names_w .*>) setfield_gc(p26, ConstPtr(ptr22), descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_strategy .*>) + setfield_gc(p26, ConstPtr(null), descr=<FieldP pypy.objspace.std.listobject.W_ListObject.inst_lstorage .*>) setarrayitem_gc(p24, 0, p26, descr=<ArrayP .>) setfield_gc(p22, p24, descr=<FieldP .*Arguments.inst_arguments_w .*>) }}} diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py --- a/pypy/module/pypyjit/test_pypy_c/test_containers.py +++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py @@ -68,10 +68,13 @@ guard_no_exception(descr=...) i12 = call(ConstClass(ll_strhash), p10, descr=<Calli . r EF=0>) p13 = new(descr=...) - p15 = new_array(8, descr=<ArrayX .*>) + p15 = new_array_clear(8, descr=<ArrayX .*>) setfield_gc(p13, p15, descr=<FieldP dicttable.entries .*>) i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<Calli . rri EF=4 OS=4>) + {{{ setfield_gc(p13, 16, descr=<FieldS dicttable.resize_counter .*>) + setfield_gc(p13, 0, descr=<FieldS dicttable.num_items .+>) + }}} guard_no_exception(descr=...) p20 = new_with_vtable(ConstClass(W_IntObject)) call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=4>) diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py --- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py +++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py @@ -1,4 +1,4 @@ -import py, sys +import py, sys, re from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC class TestCProfile(BaseTestPyPyC): @@ -26,10 +26,20 @@ for method in ['append', 'pop']: loop, = log.loops_by_id(method) print loop.ops_by_id(method) - # on 32-bit, there is f1=read_timestamp(); ...; - # f2=read_timestamp(); f3=call(llong_sub,f1,f2) - # which should turn into a single PADDQ/PSUBQ - if sys.maxint != 2147483647: - assert ' call(' not in repr(loop.ops_by_id(method)) + # on 32-bit, there is f1=call(read_timestamp); ...; + # f2=call(read_timestamp); f3=call(llong_sub,f1,f2) + # but all calls can be special-cased by the backend if + # supported. On 64-bit there is only the two calls to + # read_timestamp. + r = re.compile(r" call[(]ConstClass[(](.+?)[)]") + calls = r.findall(repr(loop.ops_by_id(method))) + if sys.maxint == 2147483647: + assert len(calls) == 6 + else: + assert len(calls) == 2 + for x in calls: + assert ('ll_read_timestamp' in x or 'llong_sub' in x + or 'llong_add' in x) + # assert ' call_may_force(' not in repr(loop.ops_by_id(method)) assert ' cond_call(' in repr(loop.ops_by_id(method)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -340,30 +340,19 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=<Callr . EF=4>) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=4 OS=110>) - i169 = int_add(i168, i97) - i170 = int_sub(i160, i106) - setfield_gc(p167, i168, descr=<FieldU pypy.module._cffi_backend.cdataobj.W_CData.inst__cdata .>) + i112 = int_sub(i160, -32768) setfield_gc(p167, ConstPtr(null), descr=<FieldP pypy.module._cffi_backend.cdataobj.W_CData.inst__lifeline_ .+>) - setfield_gc(p167, ConstPtr(ptr89), descr=<FieldP pypy.module._cffi_backend.cdataobj.W_CData.inst_ctype .+>) - i171 = uint_gt(i170, i108) - guard_false(i171, descr=...) - i172 = int_sub(i160, -32768) - i173 = int_and(i172, 65535) - i174 = int_add(i173, -32768) - setarrayitem_raw(i169, 0, i174, descr=<ArrayS 2>) - i175 = int_add(i168, i121) - i176 = int_sub(i160, i130) - i177 = uint_gt(i176, i132) - guard_false(i177, descr=...) - setarrayitem_raw(i175, 0, i174, descr=<ArrayS 2>) - i178 = int_add(i168, i140) - i179 = int_sub(i160, i149) - i180 = uint_gt(i179, i151) - guard_false(i180, descr=...) - setarrayitem_raw(i178, 0, i174, descr=<ArrayS 2>) + setfield_gc(p167, ConstPtr(ptr85), descr=<FieldP pypy.module._cffi_backend.cdataobj.W_CData.inst_ctype .+>) + i114 = uint_gt(i112, 65535) + guard_false(i114, descr=...) + i115 = int_and(i112, 65535) + i116 = int_add(i115, -32768) --TICK-- - i183 = arraylen_gc(p67, descr=<ArrayP .>) - i184 = arraylen_gc(p92, descr=<ArrayP .>) + i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=4 OS=110>) + raw_store(i119, 0, i116, descr=<ArrayS 2>) + raw_store(i119, 2, i116, descr=<ArrayS 2>) + raw_store(i119, 4, i116, descr=<ArrayS 2>) + setfield_gc(p167, i119, descr=<FieldU pypy.module._cffi_backend.cdataobj.W_CData.inst__cdata .+>) + i123 = arraylen_gc(p67, descr=<ArrayP .>) jump(..., descr=...) """) diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py --- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py +++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py @@ -1,4 +1,5 @@ from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC +from rpython.rlib.rawstorage import misaligned_is_fine class TestMicroNumPy(BaseTestPyPyC): @@ -15,6 +16,14 @@ log = self.run(main, []) assert log.result == 0 loop, = log.loops_by_filename(self.filepath) + if misaligned_is_fine: + alignment_check = "" + else: + alignment_check = """ + i93 = int_and(i79, 7) + i94 = int_is_zero(i93) + guard_true(i94, descr=...) + """ assert loop.match(""" i76 = int_lt(i71, 300) guard_true(i76, descr=...) @@ -22,6 +31,7 @@ guard_false(i77, descr=...) i78 = int_mul(i71, i61) i79 = int_add(i55, i78) + """ + alignment_check + """ f80 = raw_load(i67, i79, descr=<ArrayF 8>) i81 = int_add(i71, 1) guard_not_invalidated(descr=...) @@ -44,6 +54,14 @@ log = self.run(main, []) assert log.result == 0 loop, = log.loops_by_filename(self.filepath) + if misaligned_is_fine: + alignment_check = "" + else: + alignment_check = """ + i97 = int_and(i84, 7) + i98 = int_is_zero(i97) + guard_true(i98, descr=...) + """ assert loop.match(""" i81 = int_lt(i76, 300) guard_true(i81, descr=...) @@ -51,6 +69,7 @@ guard_false(i82, descr=...) i83 = int_mul(i76, i64) i84 = int_add(i58, i83) + """ + alignment_check + """ f85 = raw_load(i70, i84, descr=<ArrayF 8>) guard_not_invalidated(descr=...) f86 = float_add(f74, f85) diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py --- a/pypy/module/pypyjit/test_pypy_c/test_string.py +++ b/pypy/module/pypyjit/test_pypy_c/test_string.py @@ -110,9 +110,12 @@ i85 = strlen(p80) p86 = new(descr=<SizeDescr .+>) p88 = newstr(23) - setfield_gc(..., descr=<Field. stringbuilder.+>) - setfield_gc(..., descr=<Field. stringbuilder.+>) - setfield_gc(..., descr=<Field. stringbuilder.+>) + {{{ + setfield_gc(p86, 0, descr=<FieldS stringbuilder.current_pos .+>) + setfield_gc(p86, p88, descr=<FieldP stringbuilder.current_buf .+>) + setfield_gc(p86, 23, descr=<FieldS stringbuilder.current_end .+>) + setfield_gc(p86, 23, descr=<FieldS stringbuilder.total_size .+>) + }}} call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=<Callv 0 rr EF=4>) guard_no_exception(descr=...) i89 = getfield_gc(p86, descr=<FieldS stringbuilder.current_pos .+>) diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py --- a/pypy/module/select/interp_select.py +++ b/pypy/module/select/interp_select.py @@ -173,9 +173,9 @@ On Windows, only sockets are supported; on Unix, all file descriptors. """ - iwtd_w = space.listview(w_iwtd) - owtd_w = space.listview(w_owtd) - ewtd_w = space.listview(w_ewtd) + iwtd_w = space.unpackiterable(w_iwtd) + owtd_w = space.unpackiterable(w_owtd) + ewtd_w = space.unpackiterable(w_ewtd) if space.is_w(w_timeout, space.w_None): timeout = -1.0 diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -85,17 +85,18 @@ assert owtd == [writeend] total_out += writeend.send(b'x' * 512) total_in = 0 - while True: - iwtd, owtd, ewtd = select.select([readend], [], [], 0) + while total_in < total_out: + iwtd, owtd, ewtd = select.select([readend], [], [], 5) assert owtd == ewtd == [] - if iwtd == []: - break - assert iwtd == [readend] + assert iwtd == [readend] # there is more expected data = readend.recv(4096) assert len(data) > 0 assert data == b'x' * len(data) total_in += len(data) assert total_in == total_out + iwtd, owtd, ewtd = select.select([readend], [], [], 0) + assert owtd == ewtd == [] + assert iwtd == [] # there is not more expected finally: writeend.close() readend.close() @@ -304,6 +305,20 @@ for fd in rfds: os.close(fd) + def test_resize_list_in_select(self): + import select + class Foo(object): + def fileno(self): + print len(l) + if len(l) < 100: + l.append(Foo()) + return 0 + l = [Foo()] + select.select(l, (), (), 0) + assert 1 <= len(l) <= 100 + # ^^^ CPython gives 100, PyPy gives 1. I think both are OK as + # long as there is no crash. + class AppTestSelectWithSockets(_AppTestSelect): """Same tests with connected sockets. diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -18,6 +18,13 @@ _WIN32 = sys.platform == 'win32' +def _exists_and_is_executable(fn): + # os.access checks using the user's real uid and gid. + # Since pypy should not be run setuid/setgid, this + # should be sufficient. + return os.path.isfile(fn) and os.access(fn, os.X_OK) + + def find_executable(executable): """ Return the absolute path of the executable, by looking into PATH and @@ -34,14 +41,14 @@ if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) - if os.path.isfile(fn): + if _exists_and_is_executable(fn): executable = fn break executable = rpath.rabspath(executable) # 'sys.executable' should not end up being an non-existing file; # just use '' in this case. (CPython issue #7774) - return executable if os.path.isfile(executable) else '' + return executable if _exists_and_is_executable(executable) else '' def _readlink_maybe(filename): diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -7,15 +7,15 @@ # ____________________________________________________________ # -class State: - def __init__(self, space): - self.space = space +class State: + def __init__(self, space): + self.space = space self.w_modules = space.newdict(module=True) - self.w_warnoptions = space.newlist([]) self.w_argv = space.newlist([]) - self.setinitialpath(space) + + self.setinitialpath(space) def setinitialpath(self, space): from pypy.module.sys.initpath import compute_stdlib_path @@ -25,10 +25,10 @@ path = compute_stdlib_path(self, srcdir) self.w_path = space.newlist([space.wrap(p) for p in path]) - def get(space): _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
