Author: Amaury Forgeot d'Arc <[email protected]>
Branch: merge-2.7.2
Changeset: r51958:7a82f3fbab26
Date: 2012-01-29 21:55 +0000
http://bitbucket.org/pypy/pypy/changeset/7a82f3fbab26/
Log: Add large file support to mmap: 64bit offsets on 32bit platform.
problem: test_large_offset() fails when interpreted, and passes when
run on a translated pypy. Maybe a ll2ctypes issue?
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,9 +2,14 @@
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.typedef import TypeDef
from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
-from pypy.rlib import rmmap
+from pypy.rlib import rmmap, rarithmetic
from pypy.rlib.rmmap import RValueError, RTypeError
+if rmmap.HAVE_LARGEFILE_SUPPORT:
+ OFF_T = rarithmetic.r_longlong
+else:
+ OFF_T = int
+
class W_MMap(Wrappable):
def __init__(self, space, mmap_obj):
@@ -55,7 +60,7 @@
end = space.getindex_w(w_end, None)
return space.wrap(self.mmap.find(tofind, start, end, True))
- @unwrap_spec(pos='index', whence=int)
+ @unwrap_spec(pos=OFF_T, whence=int)
def seek(self, pos, whence=0):
try:
self.mmap.seek(pos, whence)
@@ -196,8 +201,8 @@
if rmmap._POSIX:
- @unwrap_spec(fileno=int, length='index', flags=int,
- prot=int, access=int, offset='index')
+ @unwrap_spec(fileno=int, length=int, flags=int,
+ prot=int, access=int, offset=OFF_T)
def mmap(space, w_subtype, fileno, length, flags=rmmap.MAP_SHARED,
prot=rmmap.PROT_WRITE | rmmap.PROT_READ,
access=rmmap._ACCESS_DEFAULT, offset=0):
@@ -216,8 +221,8 @@
elif rmmap._MS_WINDOWS:
- @unwrap_spec(fileno=int, length='index', tagname=str,
- access=int, offset='index')
+ @unwrap_spec(fileno=int, length=int, tagname=str,
+ access=int, offset=OFF_T)
def mmap(space, w_subtype, fileno, length, tagname="",
access=rmmap._ACCESS_DEFAULT, offset=0):
self = space.allocate_instance(W_MMap, w_subtype)
diff --git a/pypy/module/mmap/test/test_mmap.py
b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -591,6 +591,45 @@
assert m.size() == halfsize + 512
m.close()
+ def test_large_offset(self):
+ import mmap
+ import sys
+ size = 0x14FFFFFFF
+ if sys.platform.startswith('win') or sys.platform == 'darwin':
+ self.skip('test requires %s bytes and a long time to run' % size)
+
+ with open(self.tmpname, "w+b") as f:
+ f.seek(size)
+ f.write("A")
+ f.flush()
+ with open(self.tmpname, 'rb') as f2:
+ f2.seek(size)
+ c = f2.read(1)
+ assert c == b'A'
+ m = mmap.mmap(f2.fileno(), 0, offset=0x140000000,
+ access=mmap.ACCESS_READ)
+ try:
+ assert m[0xFFFFFFF] == b'A'
+ finally:
+ m.close()
+
+ def test_large_filesize(self):
+ import mmap
+ import sys
+ size = 0x17FFFFFFF
+ if sys.platform.startswith('win') or sys.platform == 'darwin':
+ self.skip('test requires %s bytes and a long time to run' % size)
+
+ with open(self.tmpname, "w+b") as f:
+ f.seek(size)
+ f.write(" ")
+ f.flush()
+ m = mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ)
+ try:
+ assert m.size() == 0x180000000
+ finally:
+ m.close()
+
def test_all(self):
# this is a global test, ported from test_mmap.py
import mmap
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -92,6 +92,11 @@
_ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4)
+if rffi.sizeof(off_t) > rffi.sizeof(lltype.Signed):
+ HAVE_LARGEFILE_SUPPORT = True
+else:
+ HAVE_LARGEFILE_SUPPORT = False
+
def external(name, args, result):
unsafe = rffi.llexternal(name, args, result,
compilation_info=CConfig._compilation_info_)
@@ -428,10 +433,6 @@
elif _POSIX:
st = os.fstat(self.fd)
size = st[stat.ST_SIZE]
- if size > sys.maxint:
- size = sys.maxint
- else:
- size = int(size)
return size
def write(self, data):
@@ -643,17 +644,15 @@
else:
mode = st[stat.ST_MODE]
size = st[stat.ST_SIZE]
- size -= offset
- if size > sys.maxint:
- size = sys.maxint
- else:
- size = int(size)
if stat.S_ISREG(mode):
if map_size == 0:
- if offset > st[stat.ST_SIZE]:
- raise RValueError("mmap offset is greater than file
size")
- map_size = size
- elif map_size > size:
+ if offset > size:
+ raise RValueError(
+ "mmap offset is greater than file size")
+ map_size = int(size - offset)
+ if map_size != size - offset:
+ raise RValueError("mmap length is too large")
+ elif offset + map_size > size:
raise RValueError("mmap length is greater than file size")
m = MMap(access, offset)
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py
b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -202,7 +202,7 @@
return cls._ptrtype
# ctypes can raise OverflowError on 64-bit builds
for n in [sys.maxint, 2**31]:
- cls.MAX_SIZE = n/64
+ cls.MAX_SIZE = n / ctypes.sizeof(ctypes_item)
try:
cls._ptrtype = ctypes.POINTER(cls.MAX_SIZE * ctypes_item)
except OverflowError, e:
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
