Author: Manuel Jacob <m...@manueljacob.de>
Branch:
Changeset: r90517:2d9e1b4a2e61
Date: 2017-03-03 23:21 +0100
http://bitbucket.org/pypy/pypy/changeset/2d9e1b4a2e61/
Log: Add optimized "zero-copy" path for io.FileIO.readinto().
This path is taken if the passed buffer has a raw address and is
bigger than 64 bytes.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -504,14 +504,16 @@
exception_name=exception_name,
w_exception_class=w_exception_class)
-def exception_from_saved_errno(space, w_type):
- from rpython.rlib.rposix import get_saved_errno
-
- errno = get_saved_errno()
+def exception_from_errno(space, w_type, errno):
msg = os.strerror(errno)
w_error = space.call_function(w_type, space.newint(errno),
space.newtext(msg))
return OperationError(w_type, w_error)
+def exception_from_saved_errno(space, w_type):
+ from rpython.rlib.rposix import get_saved_errno
+ errno = get_saved_errno()
+ return exception_from_errno(space, w_type, errno)
+
def new_exception_class(space, name, w_bases=None, w_dict=None):
"""Create a new exception type.
@param name: the name of the type.
diff --git a/pypy/module/_io/interp_bufferedio.py
b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,7 +4,8 @@
from pypy.interpreter.typedef import (
TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
+from rpython.rlib.rgc import (
+ nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
from rpython.rlib.buffer import Buffer
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rarithmetic import r_longlong, intmask
@@ -162,7 +163,8 @@
raise oefmt(space.w_ValueError,
"buffer size must be strictly positive")
- self.buffer = ['\0'] * self.buffer_size
+ self.buffer = resizable_list_supporting_raw_ptr(['\0'] *
+ self.buffer_size)
self.lock = TryLock(space)
@@ -561,7 +563,7 @@
if n <= current_size:
return self._read_fast(n)
- result_buffer = ['\0'] * n
+ result_buffer = resizable_list_supporting_raw_ptr(['\0'] * n)
remaining = n
written = 0
if current_size:
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -1,9 +1,11 @@
from pypy.interpreter.typedef import TypeDef, interp_attrproperty,
GetSetProperty
from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.error import (
- OperationError, oefmt, wrap_oserror, wrap_oserror2)
+ OperationError, oefmt, wrap_oserror, wrap_oserror2, exception_from_errno)
from rpython.rlib.rarithmetic import r_longlong
+from rpython.rlib.rposix import get_saved_errno
from rpython.rlib.rstring import StringBuilder
+from rpython.rtyper.lltypesystem import lltype, rffi
from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC
import sys, os, stat, errno
from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
@@ -112,6 +114,15 @@
return currentsize + BIGCHUNK
return currentsize + SMALLCHUNK
+
+_WIN32 = sys.platform.startswith('win')
+UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
+
+os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
+ [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
+ rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
+
+
class W_FileIO(W_RawIOBase):
def __init__(self, space):
W_RawIOBase.__init__(self, space)
@@ -368,15 +379,38 @@
self._check_readable(space)
rwbuffer = space.getarg_w('w*', w_buffer)
length = rwbuffer.getlength()
- try:
- buf = os.read(self.fd, length)
- except OSError as e:
- if e.errno == errno.EAGAIN:
- return space.w_None
- raise wrap_oserror(space, e,
- exception_name='w_IOError')
- rwbuffer.setslice(0, buf)
- return space.newint(len(buf))
+
+ target_address = lltype.nullptr(rffi.CCHARP.TO)
+ if length > 64:
+ try:
+ target_address = rwbuffer.get_raw_address()
+ except ValueError:
+ pass
+
+ if not target_address:
+ # unoptimized case
+ try:
+ buf = os.read(self.fd, length)
+ except OSError as e:
+ if e.errno == errno.EAGAIN:
+ return space.w_None
+ raise wrap_oserror(space, e,
+ exception_name='w_IOError')
+ rwbuffer.setslice(0, buf)
+ return space.newint(len(buf))
+ else:
+ # optimized case: reading more than 64 bytes into a rwbuffer
+ # with a valid raw address
+ got = os_read(self.fd, target_address, length)
+ got = rffi.cast(lltype.Signed, got)
+ if got >= 0:
+ return space.newint(got)
+ else:
+ err = get_saved_errno()
+ if err == errno.EAGAIN:
+ return space.w_None
+ raise exception_from_errno(space, space.w_IOError, err)
+ keepalive_until_here(rwbuffer)
def readall_w(self, space):
self._check_closed(space)
diff --git a/pypy/module/_io/test/test_fileio.py
b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -1,3 +1,4 @@
+from pypy.interpreter.gateway import interp2app
from rpython.tool.udir import udir
import os
@@ -13,6 +14,11 @@
self.w_posix = self.space.appexec([], """():
import %s as m;
return m""" % os.name)
+ def create_bigfile_w():
+ bigfile = udir.join('bigfile')
+ bigfile.write('a' * 1000, mode='wb')
+ return self.space.wrap(str(bigfile))
+ self.w_create_bigfile = self.space.wrap(interp2app(create_bigfile_w))
def test_constructor(self):
import _io
@@ -154,6 +160,13 @@
f.close()
assert a == 'a\nbxxxxxxx'
+ def test_readinto_optimized(self):
+ import _io
+ a = bytearray('x' * 1024)
+ f = _io.FileIO(self.create_bigfile(), 'r+')
+ assert f.readinto(a) == 1000
+ assert a == 'a' * 1000 + 'x' * 24
+
def test_nonblocking_read(self):
try:
import os, fcntl
@@ -169,6 +182,8 @@
assert f.read(10) is None
a = bytearray('x' * 10)
assert f.readinto(a) is None
+ a2 = bytearray('x' * 1024)
+ assert f.readinto(a2) is None
def test_repr(self):
import _io
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
