Memmapped arrays don't pickle right. I know that to get them to really pickle and restore identically, we would need some effort. However, in the current status, pickling and restoring a memmapped array leads to tracebacks that seem like they could be avoided.
I am attaching a patch with a test that shows the problem, and a fix. Should I create a ticket, or is this light-enough to be applied immediatly? Cheers, Gaƫl
Index: doc/release/1.3.0-notes.rst =================================================================== --- doc/release/1.3.0-notes.rst (revision 8436) +++ doc/release/1.3.0-notes.rst (working copy) @@ -147,6 +147,12 @@ will install a numpy which works on any x86, even if the running computer supports SSE set. +Memapping in np.load +~~~~~~~~~~~~~~~~~~~~ + +np.load now support an optional argument to do transparent memory mapping +of npy files. + Deprecated features =================== Index: numpy/core/tests/test_memmap.py =================================================================== --- numpy/core/tests/test_memmap.py (revision 8436) +++ numpy/core/tests/test_memmap.py (working copy) @@ -1,9 +1,10 @@ from tempfile import NamedTemporaryFile, mktemp import os import warnings +import pickle from numpy import memmap -from numpy import arange, allclose +from numpy import arange, allclose, all from numpy.testing import * class TestMemmap(TestCase): @@ -90,5 +91,17 @@ finally: memmap._close = _close + def test_pickle(self): + # Make sure that the memmap pickles without raising an exception + fp = memmap(self.tmpfp, dtype=self.dtype, mode='w+', + shape=self.shape) + fp[:] = self.data[:] + s = pickle.dumps(fp) + fp1 = pickle.loads(s) + # The copy can fail in the array_finalize + fp2 = fp1.copy() + self.assertTrue(all(fp == fp1)) + + if __name__ == "__main__": run_module_suite() Index: numpy/core/memmap.py =================================================================== --- numpy/core/memmap.py (revision 8436) +++ numpy/core/memmap.py (working copy) @@ -244,9 +244,12 @@ def __array_finalize__(self, obj): if hasattr(obj, '_mmap'): self._mmap = obj._mmap - self.filename = obj.filename - self.offset = obj.offset - self.mode = obj.mode + if hasattr(obj, 'filename'): + self.filename = obj.filename + if hasattr(obj, 'offset'): + self.offset = obj.offset + if hasattr(obj, 'mode'): + self.mode = obj.mode else: self._mmap = None
_______________________________________________ NumPy-Discussion mailing list NumPy-Discussion@scipy.org http://mail.scipy.org/mailman/listinfo/numpy-discussion