Author: Manuel Jacob Branch: refactor-str-types Changeset: r65607:c844d22e5405 Date: 2013-07-24 15:54 +0200 http://bitbucket.org/pypy/pypy/changeset/c844d22e5405/
Log: hg merge default diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py --- a/lib-python/2.7/test/test_os.py +++ b/lib-python/2.7/test/test_os.py @@ -275,7 +275,7 @@ try: result.f_bfree = 1 self.fail("No exception thrown") - except TypeError: + except (TypeError, AttributeError): pass try: diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -30,7 +30,7 @@ * update README * change the tracker to have a new release tag to file bugs against * go to pypy/tool/release and run: - force-builds.py /release/<release branch> + force-builds.py <release branch> * wait for builds to complete, make sure there are no failures * upload binaries to https://bitbucket.org/pypy/pypy/downloads diff --git a/pypy/doc/release-2.1.0-beta2.rst b/pypy/doc/release-2.1.0-beta2.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.1.0-beta2.rst @@ -0,0 +1,60 @@ +=============== +PyPy 2.1 beta 2 +=============== + +We're pleased to announce the second beta of the upcoming 2.1 release of PyPy. +This beta does not add any new features to the 2.1 release, but contains several bugfixes listed below. + +Highlights +========== + +* Fixed issue `1533`_: fix an RPython-level OverflowError for space.float_w(w_big_long_number). + +* Fixed issue `1552`_: GreenletExit should inherit from BaseException + +* Fixed issue `1537`_: numpypy __array_interface__ + +* Fixed issue `1238`_: Writing to an SSL socket in pypy sometimes failed with a "bad write retry" message. + +* `distutils`_: copy CPython's implementation of customize_compiler, dont call + split on environment variables, honour CFLAGS, CPPFLAGS, LDSHARED and + LDFLAGS. + +* During packaging, compile the CFFI tk extension. + +.. _`1533`: https://bugs.pypy.org/issue1533 +.. _`1552`: https://bugs.pypy.org/issue1552 +.. _`1537`: https://bugs.pypy.org/issue1537 +.. _`1238`: https://bugs.pypy.org/issue1238 +.. _`distutils`: https://bitbucket.org/pypy/pypy/src/0c6eeae0316c11146f47fcf83e21e24f11378be1/?at=distutils-cppldflags + + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7.3. It's fast due to its integrated tracing JIT compiler. + +This release supports x86 machines running Linux 32/64, Mac OS X 64 or Windows +32. Also this release supports ARM machines running Linux 32bit - anything with +``ARMv6`` (like the Raspberry Pi) or ``ARMv7`` (like Beagleboard, +Chromebook, Cubieboard, etc.) that supports ``VFPv3`` should work. + +Windows 64 work is still stalling, we would welcome a volunteer +to handle that. + +How to use PyPy? +================ + +We suggest using PyPy from a `virtualenv`_. Once you have a virtualenv +installed, you can follow instructions from `pypy documentation`_ on how +to proceed. This document also covers other `installation schemes`_. + +.. _`pypy documentation`: http://doc.pypy.org/en/latest/getting-started.html#installing-using-virtualenv +.. _`virtualenv`: http://www.virtualenv.org/en/latest/ +.. _`installation schemes`: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy +.. _`PyPy and pip`: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy + + +Cheers, +The PyPy Team. diff --git a/pypy/module/_ffi/test/test_type_converter.py b/pypy/module/_ffi/test/test_type_converter.py --- a/pypy/module/_ffi/test/test_type_converter.py +++ b/pypy/module/_ffi/test/test_type_converter.py @@ -150,7 +150,7 @@ return self.do_and_wrap(w_ffitype) -class TestFromAppLevel(object): +class TestToAppLevel(object): spaceconfig = dict(usemodules=('_ffi',)) def setup_class(cls): diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -17,12 +17,13 @@ 'setregid', 'setreuid', 'setsid', 'setuid', 'stat_float_times', 'statvfs', 'statvfs_result', 'symlink', 'sysconf', 'sysconf_names', 'tcgetpgrp', 'tcsetpgrp', 'ttyname', 'uname', 'wait', 'wait3', 'wait4' - ] +] # the Win32 urandom implementation isn't going to translate on JVM or CLI so # we have to remove it lltype_only_defs.append('urandom') + class Module(MixedModule): """This module provides access to operating system functionality that is standardized by the C Standard and the POSIX standard (a thinly @@ -32,21 +33,21 @@ applevel_name = os.name appleveldefs = { - 'error' : 'app_posix.error', - 'stat_result': 'app_posix.stat_result', - 'statvfs_result': 'app_posix.statvfs_result', - 'fdopen' : 'app_posix.fdopen', - 'tmpfile' : 'app_posix.tmpfile', - 'popen' : 'app_posix.popen', - 'tmpnam' : 'app_posix.tmpnam', - 'tempnam' : 'app_posix.tempnam', + 'error': 'app_posix.error', + 'stat_result': 'app_posix.stat_result', + 'statvfs_result': 'app_posix.statvfs_result', + 'fdopen': 'app_posix.fdopen', + 'tmpfile': 'app_posix.tmpfile', + 'popen': 'app_posix.popen', + 'tmpnam': 'app_posix.tmpnam', + 'tempnam': 'app_posix.tempnam', } if os.name == 'nt': appleveldefs.update({ - 'popen2' : 'app_posix.popen2', - 'popen3' : 'app_posix.popen3', - 'popen4' : 'app_posix.popen4', - }) + 'popen2': 'app_posix.popen2', + 'popen3': 'app_posix.popen3', + 'popen4': 'app_posix.popen4', + }) if hasattr(os, 'wait'): appleveldefs['wait'] = 'app_posix.wait' @@ -54,44 +55,46 @@ appleveldefs['wait3'] = 'app_posix.wait3' if hasattr(os, 'wait4'): appleveldefs['wait4'] = 'app_posix.wait4' - + interpleveldefs = { - 'open' : 'interp_posix.open', - 'lseek' : 'interp_posix.lseek', - 'write' : 'interp_posix.write', - 'isatty' : 'interp_posix.isatty', - 'read' : 'interp_posix.read', - 'close' : 'interp_posix.close', - 'closerange': 'interp_posix.closerange', - 'fstat' : 'interp_posix.fstat', - 'stat' : 'interp_posix.stat', - 'lstat' : 'interp_posix.lstat', - 'stat_float_times' : 'interp_posix.stat_float_times', - 'dup' : 'interp_posix.dup', - 'dup2' : 'interp_posix.dup2', - 'access' : 'interp_posix.access', - 'times' : 'interp_posix.times', - 'system' : 'interp_posix.system', - 'unlink' : 'interp_posix.unlink', - 'remove' : 'interp_posix.remove', - 'getcwd' : 'interp_posix.getcwd', - 'getcwdu' : 'interp_posix.getcwdu', - 'chdir' : 'interp_posix.chdir', - 'mkdir' : 'interp_posix.mkdir', - 'rmdir' : 'interp_posix.rmdir', - 'environ' : 'interp_posix.get(space).w_environ', - 'listdir' : 'interp_posix.listdir', - 'strerror' : 'interp_posix.strerror', - 'pipe' : 'interp_posix.pipe', - 'chmod' : 'interp_posix.chmod', - 'rename' : 'interp_posix.rename', - 'umask' : 'interp_posix.umask', - '_exit' : 'interp_posix._exit', - 'utime' : 'interp_posix.utime', - '_statfields': 'interp_posix.getstatfields(space)', - 'kill' : 'interp_posix.kill', - 'abort' : 'interp_posix.abort', - 'urandom' : 'interp_posix.urandom', + 'open': 'interp_posix.open', + 'lseek': 'interp_posix.lseek', + 'write': 'interp_posix.write', + 'isatty': 'interp_posix.isatty', + 'read': 'interp_posix.read', + 'close': 'interp_posix.close', + 'closerange': 'interp_posix.closerange', + + 'fstat': 'interp_posix.fstat', + 'stat': 'interp_posix.stat', + 'lstat': 'interp_posix.lstat', + 'stat_float_times': 'interp_posix.stat_float_times', + + 'dup': 'interp_posix.dup', + 'dup2': 'interp_posix.dup2', + 'access': 'interp_posix.access', + 'times': 'interp_posix.times', + 'system': 'interp_posix.system', + 'unlink': 'interp_posix.unlink', + 'remove': 'interp_posix.remove', + 'getcwd': 'interp_posix.getcwd', + 'getcwdu': 'interp_posix.getcwdu', + 'chdir': 'interp_posix.chdir', + 'mkdir': 'interp_posix.mkdir', + 'rmdir': 'interp_posix.rmdir', + 'environ': 'interp_posix.get(space).w_environ', + 'listdir': 'interp_posix.listdir', + 'strerror': 'interp_posix.strerror', + 'pipe': 'interp_posix.pipe', + 'chmod': 'interp_posix.chmod', + 'rename': 'interp_posix.rename', + 'umask': 'interp_posix.umask', + '_exit': 'interp_posix._exit', + 'utime': 'interp_posix.utime', + '_statfields': 'interp_posix.getstatfields(space)', + 'kill': 'interp_posix.kill', + 'abort': 'interp_posix.abort', + 'urandom': 'interp_posix.urandom', } if hasattr(os, 'chown'): @@ -168,9 +171,9 @@ interpleveldefs['getlogin'] = 'interp_posix.getlogin' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', - 'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp', - 'setpgrp', 'getppid', 'getpgid', 'setpgid', 'setreuid', - 'setregid', 'getsid', 'setsid']: + 'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp', + 'setpgrp', 'getppid', 'getpgid', 'setpgid', 'setreuid', + 'setregid', 'getsid', 'setsid', 'fstatvfs', 'statvfs']: if hasattr(os, name): interpleveldefs[name] = 'interp_posix.%s' % (name,) # not visible via os, inconsistency in nt: @@ -178,7 +181,7 @@ interpleveldefs['_getfullpathname'] = 'interp_posix._getfullpathname' if hasattr(os, 'chroot'): interpleveldefs['chroot'] = 'interp_posix.chroot' - + for name in RegisterOs.w_star: if hasattr(os, name): interpleveldefs[name] = 'interp_posix.' + name @@ -187,7 +190,7 @@ # if it's an ootype translation, remove all the defs that are lltype # only backend = space.config.translation.backend - if backend == 'cli' or backend == 'jvm': + if backend == 'cli' or backend == 'jvm' : for name in lltype_only_defs: self.interpleveldefs.pop(name, None) MixedModule.__init__(self, space, w_name) @@ -195,7 +198,7 @@ def startup(self, space): from pypy.module.posix import interp_posix interp_posix.get(space).startup(space) - + for constant in dir(os): value = getattr(os, constant) if constant.isupper() and type(value) is int: diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -1,15 +1,17 @@ -from pypy.interpreter.gateway import unwrap_spec +import os +import sys + from rpython.rlib import rposix, objectmodel, rurandom from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.unroll import unrolling_iterable +from rpython.rtyper.module import ll_os_stat +from rpython.rtyper.module.ll_os import RegisterOs + +from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 -from rpython.rtyper.module.ll_os import RegisterOs -from rpython.rtyper.module import ll_os_stat from pypy.module.sys.interp_encoding import getfilesystemencoding -import os -import sys _WIN32 = sys.platform == 'win32' if _WIN32: @@ -213,6 +215,7 @@ STAT_FIELDS = unrolling_iterable(enumerate(ll_os_stat.STAT_FIELDS)) PORTABLE_STAT_FIELDS = unrolling_iterable( enumerate(ll_os_stat.PORTABLE_STAT_FIELDS)) +STATVFS_FIELDS = unrolling_iterable(enumerate(ll_os_stat.STATVFS_FIELDS)) def build_stat_result(space, st): if space.config.translation.type_system == 'ootype': @@ -253,6 +256,16 @@ space.wrap('stat_result')) return space.call_function(w_stat_result, w_tuple, w_keywords) + +def build_statvfs_result(space, st): + vals_w = [None] * len(ll_os_stat.STATVFS_FIELDS) + for i, (name, _) in STATVFS_FIELDS: + vals_w[i] = space.wrap(getattr(st, name)) + w_tuple = space.newtuple(vals_w) + w_statvfs_result = space.getattr(space.getbuiltinmodule(os.name), space.wrap('statvfs_result')) + return space.call_function(w_statvfs_result, w_tuple) + + @unwrap_spec(fd=c_int) def fstat(space, fd): """Perform a stat system call on the file referenced to by an open @@ -314,6 +327,26 @@ else: state.stat_float_times = space.bool_w(w_value) + +@unwrap_spec(fd=c_int) +def fstatvfs(space, fd): + try: + st = os.fstatvfs(fd) + except OSError as e: + raise wrap_oserror(space, e) + else: + return build_statvfs_result(space, st) + + +def statvfs(space, w_path): + try: + st = dispatch_filename(rposix.statvfs)(space, w_path) + except OSError as e: + raise wrap_oserror2(space, e, w_path) + else: + return build_statvfs_result(space, st) + + @unwrap_spec(fd=c_int) def dup(space, fd): """Create a copy of the file descriptor. Return the new file diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -169,7 +169,8 @@ assert stat.S_ISDIR(st.st_mode) def test_stat_exception(self): - import sys, errno + import sys + import errno for fn in [self.posix.stat, self.posix.lstat]: try: fn("nonexistentdir/nonexistentfile") @@ -183,6 +184,15 @@ assert isinstance(e, WindowsError) assert e.winerror == 3 + def test_statvfs(self): + st = self.posix.statvfs(".") + assert isinstance(st, self.posix.statvfs_result) + for field in [ + 'f_bsize', 'f_frsize', 'f_blocks', 'f_bfree', 'f_bavail', + 'f_files', 'f_ffree', 'f_favail', 'f_flag', 'f_namemax', + ]: + assert hasattr(st, field) + def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -125,7 +125,12 @@ self.llbox = llbox def descr_getint(self, space): - return space.wrap(jit_hooks.box_getint(self.llbox)) + try: + value = jit_hooks.box_getint(self.llbox) + except NotImplementedError: + raise OperationError(space.w_NotImplementedError, + space.wrap("Box has no int value")) + return space.wrap(value) @unwrap_spec(no=int) def descr_new_box(space, w_tp, no): @@ -182,7 +187,12 @@ @unwrap_spec(no=int) def descr_getarg(self, space, no): - return WrappedBox(jit_hooks.resop_getarg(self.op, no)) + try: + box = jit_hooks.resop_getarg(self.op, no) + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("Index out of range")) + return WrappedBox(box) @unwrap_spec(no=int, w_box=WrappedBox) def descr_setarg(self, space, no, w_box): @@ -232,7 +242,8 @@ getarg = interp2app(WrappedOp.descr_getarg), setarg = interp2app(WrappedOp.descr_setarg), result = GetSetProperty(WrappedOp.descr_getresult, - WrappedOp.descr_setresult) + WrappedOp.descr_setresult), + offset = interp_attrproperty("offset", cls=WrappedOp), ) WrappedOp.acceptable_as_base_class = False @@ -342,6 +353,10 @@ doc="bridge number (if a bridge)"), type = interp_attrproperty('type', cls=W_JitLoopInfo, doc="Loop type"), + asmaddr = interp_attrproperty('asmaddr', cls=W_JitLoopInfo, + doc="Address of machine code"), + asmlen = interp_attrproperty('asmlen', cls=W_JitLoopInfo, + doc="Length of machine code"), __repr__ = interp2app(W_JitLoopInfo.descr_repr), ) W_JitLoopInfo.acceptable_as_base_class = False diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -71,7 +71,7 @@ greenkey) di_loop_optimize = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(), oplist, 'loop', greenkey) - di_loop.asminfo = AsmInfo(offset, 0, 0) + di_loop.asminfo = AsmInfo(offset, 0x42, 12) di_bridge = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(), oplist, 'bridge', fail_descr=BasicFailDescr()) di_bridge.asminfo = AsmInfo(offset, 0, 0) @@ -123,6 +123,8 @@ assert info.greenkey[2] == False assert info.loop_no == 0 assert info.type == 'loop' + assert info.asmaddr == 0x42 + assert info.asmlen == 12 raises(TypeError, 'info.bridge_no') assert len(info.operations) == 4 int_add = info.operations[0] @@ -132,8 +134,10 @@ assert dmp.greenkey == (self.f.func_code, 0, False) assert dmp.call_depth == 0 assert dmp.call_id == 0 + assert dmp.offset == -1 assert int_add.name == 'int_add' assert int_add.num == self.int_add_num + assert int_add.offset == 0 self.on_compile_bridge() expected = ('<JitLoopInfo pypyjit, 4 operations, starting at ' '<(%s, 0, False)>>' % repr(self.f.func_code)) @@ -160,6 +164,20 @@ assert 'jit hook' in s.getvalue() assert 'ZeroDivisionError' in s.getvalue() + def test_on_compile_crashes(self): + import pypyjit + loops = [] + def hook(loop): + loops.append(loop) + pypyjit.set_compile_hook(hook) + self.on_compile() + loop = loops[0] + op = loop.operations[2] + # Should not crash the interpreter + raises(IndexError, op.getarg, 2) + assert op.name == 'guard_nonnull' + raises(NotImplementedError, op.getarg(0).getint) + def test_non_reentrant(self): import pypyjit l = [] diff --git a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py --- a/pypy/module/test_lib_pypy/test_ctypes_config_cache.py +++ b/pypy/module/test_lib_pypy/test_ctypes_config_cache.py @@ -33,10 +33,8 @@ def test_resource(): - try: - import lib_pypy.resource - except ImportError: - py.test.skip('no syslog on this platform') + if sys.platform == 'win32': + py.test.skip('no resource module on this platform') d = run('resource.ctc.py', '_resource_cache.py') assert 'RLIM_NLIMITS' in d diff --git a/pypy/module/test_lib_pypy/test_md5_extra.py b/pypy/module/test_lib_pypy/test_md5_extra.py --- a/pypy/module/test_lib_pypy/test_md5_extra.py +++ b/pypy/module/test_lib_pypy/test_md5_extra.py @@ -1,227 +1,226 @@ """A test script to compare MD5 implementations. -A note about performance: the pure Python MD5 takes roughly -160 sec. per MB of data on a 233 MHz Intel Pentium CPU. +A note about performance: the pure Python MD5 takes roughly 160 sec. per +MB of data on a 233 MHz Intel Pentium CPU. """ +import md5 -from __future__ import absolute_import -import md5 # CPython's implementation in C. -from lib_pypy import _md5 as pymd5 +from pypy.module.test_lib_pypy.support import import_lib_pypy -# Helpers... +def compare_host(message, d2, d2h): + """Compare results against the host Python's builtin md5. -def formatHex(str): - "Print a string's HEX code in groups of two digits." - - d = map(None, str) - d = map(ord, d) - d = map(lambda x:"%02x" % x, d) - return ' '.join(d) - - -def format(str): - "Print a string as-is in groups of two characters." - - s = '' - for i in range(0, len(str)-1, 2): - s = s + "%03s" % str[i:i+2] - return s[1:] - - -def printDiff(message, d1, d2, expectedResult=None): - "Print different outputs for same message." - - print "Message: '%s'" % message - print "Message length: %d" % len(message) - if expectedResult: - print "%-48s (expected)" % format(expectedResult) - print "%-48s (Std. lib. MD5)" % formatHex(d1) - print "%-48s (Pure Python MD5)" % formatHex(d2) - print - - -# The real comparison function. - -def compareImp(message): - """Compare two MD5 implementations, C vs. pure Python module. - - For equal digests this returns None, otherwise it returns - a tuple of both digests. + For equal digests this returns None, otherwise it returns a tuple of + both digests. """ - - # Use Python's standard library MD5 compiled C module. + # Use the host Python's standard library MD5 compiled C module. m1 = md5.md5() m1.update(message) d1 = m1.digest() d1h = m1.hexdigest() - - # Use MD5 module in pure Python. - m2 = pymd5.new() - m2.update(message) - d2 = m2.digest() - d2h = m2.hexdigest() + # Return None if equal or the different digests if not equal. + return None if d1 == d2 and d1h == d2h else (d1, d2) - # Return None if equal or the different digests if not equal. - if d1 == d2 and d1h == d2h: - return - else: - return d1, d2 +class TestMD5Update: -class TestMD5Compare: - "Compare pure Python MD5 against Python's std. lib. version." - + spaceconfig = dict(usemodules=('struct',)) + + def test_update(self): + """Test updating cloned objects.""" + cases = ( + "123", + "1234", + "12345", + "123456", + "1234567", + "12345678", + "123456789 123456789 123456789 ", + "123456789 123456789 ", + "123456789 123456789 1", + "123456789 123456789 12", + "123456789 123456789 123", + "123456789 123456789 1234", + "123456789 123456789 123456789 1", + "123456789 123456789 123456789 12", + "123456789 123456789 123456789 123", + "123456789 123456789 123456789 1234", + "123456789 123456789 123456789 12345", + "123456789 123456789 123456789 123456", + "123456789 123456789 123456789 1234567", + "123456789 123456789 123456789 12345678", + ) + space = self.space + w__md5 = import_lib_pypy(space, '_md5') + + # Load both with same prefix. + prefix1 = 2**10 * 'a' + + # The host md5 + m1 = md5.md5() + m1.update(prefix1) + m1c = m1.copy() + + # The app-level _md5 + w_m2 = space.call_method(w__md5, 'new') + space.call_method(w_m2, 'update', space.wrap(prefix1)) + w_m2c = space.call_method(w_m2, 'copy') + + # Update and compare... + for i in range(len(cases)): + message = cases[i][0] + + m1c.update(message) + d1 = m1c.hexdigest() + + space.call_method(w_m2c, 'update', space.wrap(message)) + w_d2 = space.call_method(w_m2c, 'hexdigest') + d2 = space.str_w(w_d2) + + assert d1 == d2 + + +class AppTestMD5Compare: + """Compare pure Python MD5 against Python's std. lib. version.""" + + spaceconfig = dict(usemodules=('struct',)) + + def setup_class(cls): + from pypy.interpreter import gateway + space = cls.space + cls.w__md5 = import_lib_pypy(space, '_md5') + if cls.runappdirect: + # interp2app doesn't work in appdirect mode + cls.w_compare_host = staticmethod(compare_host) + else: + compare_host.unwrap_spec = [str, str, str] + cls.w_compare_host = space.wrap(gateway.interp2app(compare_host)) + + def w_compare(self, message): + # Generate results against the app-level pure Python MD5 and + # pass them off for comparison against the host Python's MD5 + m2 = self._md5.new() + m2.update(message) + return self.compare_host(message, m2.digest(), m2.hexdigest()) + + def w__format_hex(self, string): + """Print a string's HEX code in groups of two digits.""" + d = map(None, string) + d = map(ord, d) + d = map(lambda x: "%02x" % x, d) + return ' '.join(d) + + def w__format(self, string): + """Print a string as-is in groups of two characters.""" + s = '' + for i in range(0, len(string) - 1, 2): + s = s + "%03s" % string[i:i + 2] + return s[1:] + + def w_print_diff(self, message, d1, d2, expectedResult=None): + """Print different outputs for same message.""" + print("Message: '%s'" % message) + print("Message length: %d" % len(message)) + if expectedResult: + print("%-48s (expected)" % self._format(expectedResult)) + print("%-48s (Std. lib. MD5)" % self._format_hex(d1)) + print("%-48s (Pure Python MD5)" % self._format_hex(d2)) + print() + def test1(self): - "Test cases with known digest result." - + """Test cases with known digest result.""" cases = ( - ("", - "d41d8cd98f00b204e9800998ecf8427e"), - ("a", - "0cc175b9c0f1b6a831c399e269772661"), - ("abc", - "900150983cd24fb0d6963f7d28e17f72"), - ("message digest", - "f96b697d7cb7938d525a2f31aaf161d0"), - ("abcdefghijklmnopqrstuvwxyz", - "c3fcd3d76192e4007dfb496cca67e13b"), - ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", - "d174ab98d277d9f5a5611c2c9f419d9f"), - ("1234567890"*8, - "57edf4a22be3c955ac49da2e2107b67a"), - ) + ("", + "d41d8cd98f00b204e9800998ecf8427e"), + ("a", + "0cc175b9c0f1b6a831c399e269772661"), + ("abc", + "900150983cd24fb0d6963f7d28e17f72"), + ("message digest", + "f96b697d7cb7938d525a2f31aaf161d0"), + ("abcdefghijklmnopqrstuvwxyz", + "c3fcd3d76192e4007dfb496cca67e13b"), + ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f"), + ("1234567890"*8, + "57edf4a22be3c955ac49da2e2107b67a"), + ) - for i in xrange(len(cases)): - res = compareImp(cases[i][0]) + for i in range(len(cases)): + res = self.compare(cases[i][0]) if res is not None: d1, d2 = res message, expectedResult = cases[i][0], None if len(cases[i]) == 2: expectedResult = cases[i][1] - printDiff(message, d1, d2, expectedResult) + self.print_diff(message, d1, d2, expectedResult) assert res is None + def test2(self): + """Test cases without known digest result.""" + cases = ( + "123", + "1234", + "12345", + "123456", + "1234567", + "12345678", + "123456789 123456789 123456789 ", + "123456789 123456789 ", + "123456789 123456789 1", + "123456789 123456789 12", + "123456789 123456789 123", + "123456789 123456789 1234", + "123456789 123456789 123456789 1", + "123456789 123456789 123456789 12", + "123456789 123456789 123456789 123", + "123456789 123456789 123456789 1234", + "123456789 123456789 123456789 12345", + "123456789 123456789 123456789 123456", + "123456789 123456789 123456789 1234567", + "123456789 123456789 123456789 12345678", + ) - def test2(self): - "Test cases without known digest result." - - cases = ( - "123", - "1234", - "12345", - "123456", - "1234567", - "12345678", - "123456789 123456789 123456789 ", - "123456789 123456789 ", - "123456789 123456789 1", - "123456789 123456789 12", - "123456789 123456789 123", - "123456789 123456789 1234", - "123456789 123456789 123456789 1", - "123456789 123456789 123456789 12", - "123456789 123456789 123456789 123", - "123456789 123456789 123456789 1234", - "123456789 123456789 123456789 12345", - "123456789 123456789 123456789 123456", - "123456789 123456789 123456789 1234567", - "123456789 123456789 123456789 12345678", - ) - - for i in xrange(len(cases)): - res = compareImp(cases[i][0]) + for i in range(len(cases)): + res = self.compare(cases[i][0]) if res is not None: d1, d2 = res message = cases[i][0] - printDiff(message, d1, d2) + self.print_diff(message, d1, d2) assert res is None + def test3(self): + """Test cases with long messages (can take a while).""" + cases = ( + (2**10*'a',), + (2**10*'abcd',), + #(2**20*'a',), # 1 MB, takes about 160 sec. on a 233 Mhz Pentium. + ) - def test3(self): - "Test cases with long messages (can take a while)." - - cases = ( - (2**10*'a',), - (2**10*'abcd',), -## (2**20*'a',), ## 1 MB, takes about 160 sec. on a 233 Mhz Pentium. - ) - - for i in xrange(len(cases)): - res = compareImp(cases[i][0]) + for i in range(len(cases)): + res = self.compare(cases[i][0]) if res is not None: d1, d2 = res message = cases[i][0] - printDiff(message, d1, d2) + self.print_diff(message, d1, d2) assert res is None - def test4(self): - "Test cases with increasingly growing message lengths." - + """Test cases with increasingly growing message lengths.""" i = 0 - while i < 2**5: + while i < 2**5: message = i * 'a' - res = compareImp(message) + res = self.compare(message) if res is not None: d1, d2 = res - printDiff(message, d1, d2) + self.print_diff(message, d1, d2) assert res is None - i = i + 1 + i += 1 - - def test5(self): - "Test updating cloned objects." - - cases = ( - "123", - "1234", - "12345", - "123456", - "1234567", - "12345678", - "123456789 123456789 123456789 ", - "123456789 123456789 ", - "123456789 123456789 1", - "123456789 123456789 12", - "123456789 123456789 123", - "123456789 123456789 1234", - "123456789 123456789 123456789 1", - "123456789 123456789 123456789 12", - "123456789 123456789 123456789 123", - "123456789 123456789 123456789 1234", - "123456789 123456789 123456789 12345", - "123456789 123456789 123456789 123456", - "123456789 123456789 123456789 1234567", - "123456789 123456789 123456789 12345678", - ) - - # Load both with same prefix. - prefix1 = 2**10 * 'a' - - m1 = md5.md5() - m1.update(prefix1) - m1c = m1.copy() - - m2 = pymd5.new() - m2.update(prefix1) - m2c = m2.copy() - - # Update and compare... - for i in xrange(len(cases)): - message = cases[i][0] - - m1c.update(message) - d1 = m1c.hexdigest() - - m2c.update(message) - d2 = m2c.hexdigest() - - assert d1 == d2 - - -def test_attributes(): - assert pymd5.digest_size == 16 - assert pymd5.new().digest_size == 16 - assert pymd5.new().digestsize == 16 - assert pymd5.new().block_size == 64 + def test_attributes(self): + _md5 = self._md5 + assert _md5.digest_size == 16 + assert _md5.new().digest_size == 16 + assert _md5.new().digestsize == 16 + assert _md5.new().block_size == 64 diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -735,6 +735,10 @@ self.mc.RET() def _load_shadowstack_top_in_ebx(self, mc, gcrootmap): + """Loads the shadowstack top in ebx, and returns an integer + that gives the address of the stack top. If this integer doesn't + fit in 32 bits, it will be loaded in r11. + """ rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): mc.MOV_rj(ebx.value, rst) # MOV ebx, [rootstacktop] @@ -752,6 +756,9 @@ if rx86.fits_in_32bits(rst): self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], ebx else: + # The integer 'rst' doesn't fit in 32 bits, so we know that + # _load_shadowstack_top_in_ebx() above loaded it in r11. + # Reuse it. Be careful not to overwrite r11 in the middle! self.mc.MOV_mr((X86_64_SCRATCH_REG.value, 0), ebx.value) # MOV [r11], ebx diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -154,6 +154,15 @@ else: return os.lstat(path.as_bytes()) + +@specialize.argtype(0) +def statvfs(path): + if isinstance(path, str): + return os.statvfs(path) + else: + return os.statvfs(path.as_bytes()) + + @specialize.argtype(0) def unlink(path): if isinstance(path, str): diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -1698,6 +1698,18 @@ from rpython.rtyper.module import ll_os_stat return ll_os_stat.register_stat_variant('lstat', traits) + @registering_if(os, 'fstatvfs') + def register_os_fstatvfs(self): + from rpython.rtyper.module import ll_os_stat + return ll_os_stat.register_statvfs_variant('fstatvfs', StringTraits()) + + if hasattr(os, 'statvfs'): + @registering_str_unicode(os.statvfs) + def register_os_statvfs(self, traits): + from rpython.rtyper.module import ll_os_stat + return ll_os_stat.register_statvfs_variant('statvfs', traits) + + # ------------------------------- os.W* --------------------------------- w_star = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED', diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py --- a/rpython/rtyper/module/ll_os_stat.py +++ b/rpython/rtyper/module/ll_os_stat.py @@ -2,20 +2,22 @@ and os.fstat(). In RPython like in plain Python the stat result can be indexed like a tuple but also exposes the st_xxx attributes. """ -import os, sys + +import os +import sys + from rpython.annotator import model as annmodel -from rpython.tool.pairtype import pairtype -from rpython.tool.sourcetools import func_with_new_name, func_renamer -from rpython.rtyper import extregistry -from rpython.rtyper.extfunc import register_external, extdef -from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.tool import rffi_platform as platform -from rpython.rtyper.lltypesystem.rtupletype import TUPLE_TYPE from rpython.rlib import rposix from rpython.rlib.rarithmetic import intmask -from rpython.rlib.objectmodel import specialize +from rpython.rtyper import extregistry +from rpython.rtyper.annlowlevel import hlstr +from rpython.rtyper.extfunc import extdef +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.lltypesystem.rtupletype import TUPLE_TYPE +from rpython.rtyper.tool import rffi_platform as platform +from rpython.tool.pairtype import pairtype +from rpython.tool.sourcetools import func_renamer from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rtyper.annlowlevel import hlstr # Support for float times is here. # - ALL_STAT_FIELDS contains Float fields if the system can retrieve @@ -47,12 +49,26 @@ ("st_flags", lltype.Signed), #("st_gen", lltype.Signed), -- new in CPy 2.5, not implemented #("st_birthtime", lltype.Float), -- new in CPy 2.5, not implemented - ] +] N_INDEXABLE_FIELDS = 10 # For OO backends, expose only the portable fields (the first 10). PORTABLE_STAT_FIELDS = ALL_STAT_FIELDS[:N_INDEXABLE_FIELDS] +STATVFS_FIELDS = [ + ("f_bsize", lltype.Signed), + ("f_frsize", lltype.Signed), + ("f_blocks", lltype.Signed), + ("f_bfree", lltype.Signed), + ("f_bavail", lltype.Signed), + ("f_files", lltype.Signed), + ("f_ffree", lltype.Signed), + ("f_favail", lltype.Signed), + ("f_flag", lltype.Signed), + ("f_namemax", lltype.Signed), +] + + # ____________________________________________________________ # # Annotation support @@ -79,6 +95,7 @@ def stat_result_reduce(st): return (st[0], st[1], st[2], st[3], st[4], st[5], st[6], st[7], st[8], st[9]) + def stat_result_recreate(tup): return make_stat_result(tup + extra_zeroes) s_reduced = annmodel.SomeTuple([annmodel.lltype_to_annotation(TYPE) @@ -86,6 +103,26 @@ extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS)) return s_reduced, stat_result_reduce, stat_result_recreate + +class SomeStatvfsResult(annmodel.SomeObject): + if hasattr(os, 'statvfs_result'): + knowntype = os.statvfs_result + else: + knowntype = None # will not be used + + def rtyper_makerepr(self, rtyper): + from rpython.rtyper.module import r_os_stat + return r_os_stat.StatvfsResultRepr(rtyper) + + def rtyper_makekey_ex(self, rtyper): + return self.__class__, + + def getattr(self, s_attr): + assert s_attr.is_constant() + TYPE = STATVFS_FIELD_TYPES[s_attr.const] + return annmodel.lltype_to_annotation(TYPE) + + class __extend__(pairtype(SomeStatResult, annmodel.SomeInteger)): def getitem((s_sta, s_int)): assert s_int.is_constant(), "os.stat()[index]: index must be constant" @@ -94,7 +131,17 @@ name, TYPE = STAT_FIELDS[index] return annmodel.lltype_to_annotation(TYPE) + +class __extend__(pairtype(SomeStatvfsResult, annmodel.SomeInteger)): + def getitem((s_stat, s_int)): + assert s_int.is_constant() + name, TYPE = STATVFS_FIELDS[s_int.const] + return annmodel.lltype_to_annotation(TYPE) + + s_StatResult = SomeStatResult() +s_StatvfsResult = SomeStatvfsResult() + def make_stat_result(tup): """Turn a tuple into an os.stat_result object.""" @@ -104,6 +151,11 @@ kwds[name] = tup[N_INDEXABLE_FIELDS + i] return os.stat_result(positional, kwds) + +def make_statvfs_result(tup): + return os.statvfs_result(tup) + + class MakeStatResultEntry(extregistry.ExtRegistryEntry): _about_ = make_stat_result @@ -114,22 +166,33 @@ from rpython.rtyper.module import r_os_stat return r_os_stat.specialize_make_stat_result(hop) + +class MakeStatvfsResultEntry(extregistry.ExtRegistryEntry): + _about_ = make_statvfs_result + + def compute_result_annotation(self, s_tup): + return s_StatvfsResult + + def specialize_call(self, hop): + from rpython.rtyper.module import r_os_stat + return r_os_stat.specialize_make_statvfs_result(hop) + # ____________________________________________________________ # # RFFI support if sys.platform.startswith('win'): _name_struct_stat = '_stati64' - INCLUDES = ['sys/types.h', 'sys/stat.h'] + INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h'] else: _name_struct_stat = 'stat' - INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] + INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h'] compilation_info = ExternalCompilationInfo( # This must be set to 64 on some systems to enable large file support. #pre_include_bits = ['#define _FILE_OFFSET_BITS 64'], # ^^^ nowadays it's always set in all C files we produce. - includes = INCLUDES + includes=INCLUDES ) if TIMESPEC is not None: @@ -141,7 +204,7 @@ def posix_declaration(try_to_add=None): - global STAT_STRUCT + global STAT_STRUCT, STATVFS_STRUCT LL_STAT_FIELDS = STAT_FIELDS[:] if try_to_add: @@ -173,15 +236,17 @@ class CConfig: _compilation_info_ = compilation_info STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS) + STATVFS_STRUCT = platform.Struct('struct statvfs', STATVFS_FIELDS) + try: - config = platform.configure(CConfig, ignore_errors= - try_to_add is not None) + config = platform.configure(CConfig, ignore_errors=try_to_add is not None) except platform.CompilationError: if try_to_add: return # failed to add this field, give up raise STAT_STRUCT = lltype.Ptr(config['STAT_STRUCT']) + STATVFS_STRUCT = lltype.Ptr(config['STATVFS_STRUCT']) if try_to_add: STAT_FIELDS.append(try_to_add) @@ -202,6 +267,9 @@ STAT_FIELD_NAMES = [_name for (_name, _TYPE) in STAT_FIELDS] del _name, _TYPE +STATVFS_FIELD_TYPES = dict(STATVFS_FIELDS) +STATVFS_FIELD_NAMES = [name for name, tp in STATVFS_FIELDS] + def build_stat_result(st): # only for LL backends @@ -233,6 +301,21 @@ return make_stat_result(result) +def build_statvfs_result(st): + return make_statvfs_result(( + st.c_f_bsize, + st.c_f_frsize, + st.c_f_blocks, + st.c_f_bfree, + st.c_f_bavail, + st.c_f_files, + st.c_f_ffree, + st.c_f_favail, + st.c_f_flag, + st.c_f_namemax + )) + + def register_stat_variant(name, traits): if name != 'fstat': arg_is_path = True @@ -301,6 +384,56 @@ [s_arg], s_StatResult, "ll_os.ll_os_%s" % (name,), llimpl=posix_stat_llimpl, llfakeimpl=posix_fakeimpl) + +def register_statvfs_variant(name, traits): + if name != 'fstatvfs': + arg_is_path = True + s_arg = traits.str0 + ARG1 = traits.CCHARP + else: + arg_is_path = False + s_arg = int + ARG1 = rffi.INT + + posix_mystatvfs = rffi.llexternal(name, + [ARG1, STATVFS_STRUCT], rffi.INT, + compilation_info=compilation_info + ) + + @func_renamer('os_%s_llimpl' % (name,)) + def posix_statvfs_llimpl(arg): + stresult = lltype.malloc(STATVFS_STRUCT.TO, flavor='raw') + try: + if arg_is_path: + arg = traits.str2charp(arg) + error = rffi.cast(rffi.LONG, posix_mystatvfs(arg, stresult)) + if arg_is_path: + traits.free_charp(arg) + if error != 0: + raise OSError(rposix.get_errno(), "os_?statvfs failed") + return build_statvfs_result(stresult) + finally: + lltype.free(stresult, flavor='raw') + + @func_renamer('os_%s_fake' % (name,)) + def posix_fakeimpl(arg): + if s_arg == traits.str0: + arg = hlstr(arg) + st = getattr(os, name)(arg) + fields = [TYPE for fieldname, TYPE in STATVFS_FIELDS] + TP = TUPLE_TYPE(fields) + ll_tup = lltype.malloc(TP.TO) + for i, (fieldname, TYPE) in enumerate(STATVFS_FIELDS): + val = getattr(st, fieldname) + rffi.setintfield(ll_tup, 'item%d' % i, int(val)) + return ll_tup + + return extdef( + [s_arg], s_StatvfsResult, "ll_os.ll_os_%s" % (name,), + llimpl=posix_statvfs_llimpl, llfakeimpl=posix_fakeimpl + ) + + def make_win32_stat_impl(name, traits): from rpython.rlib import rwin32 from rpython.rtyper.module.ll_win32file import make_win32_traits diff --git a/rpython/rtyper/module/r_os_stat.py b/rpython/rtyper/module/r_os_stat.py --- a/rpython/rtyper/module/r_os_stat.py +++ b/rpython/rtyper/module/r_os_stat.py @@ -67,3 +67,52 @@ # no-op conversion from r_StatResult.r_tuple to r_StatResult hop.exception_cannot_occur() return v_result + + +class StatvfsResultRepr(Repr): + + def __init__(self, rtyper): + self.rtyper = rtyper + self.statvfs_fields = ll_os_stat.STATVFS_FIELDS + + self.statvfs_field_indexes = {} + for i, (name, TYPE) in enumerate(self.statvfs_fields): + self.statvfs_field_indexes[name] = i + + self.s_tuple = annmodel.SomeTuple([annmodel.lltype_to_annotation(TYPE) + for name, TYPE in self.statvfs_fields]) + self.r_tuple = rtyper.getrepr(self.s_tuple) + self.lowleveltype = self.r_tuple.lowleveltype + + def redispatch_getfield(self, hop, index): + rtyper = self.rtyper + s_index = rtyper.annotator.bookkeeper.immutablevalue(index) + hop2 = hop.copy() + hop2.forced_opname = 'getitem' + hop2.args_v = [hop2.args_v[0], Constant(index)] + hop2.args_s = [self.s_tuple, s_index] + hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] + return hop2.dispatch() + + def rtype_getattr(self, hop): + s_attr = hop.args_s[1] + attr = s_attr.const + try: + index = self.statvfs_field_indexes[attr] + except KeyError: + raise TyperError("os.statvfs().%s: field not available" % (attr,)) + return self.redispatch_getfield(hop, index) + + +class __extend__(pairtype(StatvfsResultRepr, IntegerRepr)): + def rtype_getitem((r_sta, r_int), hop): + s_int = hop.args_s[1] + index = s_int.const + return r_sta.redispatch_getfield(hop, index) + + +def specialize_make_statvfs_result(hop): + r_StatvfsResult = hop.rtyper.getrepr(ll_os_stat.s_StatvfsResult) + [v_result] = hop.inputargs(r_StatvfsResult.r_tuple) + hop.exception_cannot_occur() + return v_result diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py --- a/rpython/translator/c/gcc/trackgcroot.py +++ b/rpython/translator/c/gcc/trackgcroot.py @@ -478,7 +478,7 @@ 'rep', 'movs', 'movhp', 'lods', 'stos', 'scas', 'cwde', 'prefetch', # floating-point operations cannot produce GC pointers 'f', - 'cvt', 'ucomi', 'comi', 'subs', 'subp' , 'adds', 'addp', 'xorp', + 'cvt', 'ucomi', 'comi', 'subs', 'subp', 'adds', 'addp', 'xorp', 'movap', 'movd', 'movlp', 'movup', 'sqrt', 'rsqrt', 'movhlp', 'movlhp', 'mins', 'minp', 'maxs', 'maxp', 'unpck', 'pxor', 'por', # sse2 'shufps', 'shufpd', @@ -495,13 +495,15 @@ # sign-extending moves should not produce GC pointers 'cbtw', 'cwtl', 'cwtd', 'cltd', 'cltq', 'cqto', # zero-extending moves should not produce GC pointers - 'movz', + 'movz', # locked operations should not move GC pointers, at least so far 'lock', 'pause', # non-temporal moves should be reserved for areas containing # raw data, not GC pointers 'movnt', 'mfence', 'lfence', 'sfence', - ]) + # bit manipulations + 'bextr', + ]) # a partial list is hopefully good enough for now; it's all to support # only one corner case, tested in elf64/track_zero.s @@ -741,7 +743,7 @@ # tail-calls are equivalent to RET for us return InsnRet(self.CALLEE_SAVE_REGISTERS) return InsnStop("jump") - + def register_jump_to(self, label, lastinsn=None): if lastinsn is None: lastinsn = self.insns[-1] @@ -1020,7 +1022,7 @@ visit_movl = visit_mov visit_xorl = _maybe_32bit_dest(FunctionGcRootTracker.binary_insn) - + visit_pushq = FunctionGcRootTracker._visit_push visit_addq = FunctionGcRootTracker._visit_add _______________________________________________ pypy-commit mailing list pypy-commit@python.org http://mail.python.org/mailman/listinfo/pypy-commit