This is an automated email from the git hooks/post-receive script. yoh pushed a commit to tag 0.4 in repository python-mne.
commit 58e8c0dc27bb6e63d5b3d343951d31ee0ee0b1dd Author: Martin Luessi <[email protected]> Date: Tue Jan 3 12:37:51 2012 -0500 ENH: preload keyword in init, support memmapped-file, moved test --- mne/fiff/raw.py | 67 +++++++++++++++++++++++++++++++--------------- mne/fiff/tests/test_raw.py | 30 +++++++++++++++++++++ mne/tests/test_raw.py | 46 ------------------------------- 3 files changed, 75 insertions(+), 68 deletions(-) diff --git a/mne/fiff/raw.py b/mne/fiff/raw.py index 47a8b8b..2b99e01 100644 --- a/mne/fiff/raw.py +++ b/mne/fiff/raw.py @@ -31,7 +31,7 @@ class Raw(dict): """ - def __init__(self, fname, allow_maxshield=False): + def __init__(self, fname, allow_maxshield=False, preload=False): """ Parameters ---------- @@ -41,6 +41,12 @@ class Raw(dict): allow_maxshield: bool, (default False) allow_maxshield if True XXX ??? + preload: bool or str (default False) + Preload data into memory for data manipulation and faster indexing. + If True, the data will be preloaded into memory (fast, requires + large amount of memory). If preload is a string, preload is the + file name of a memory-mapped file which is used to store the data + (slower, requires less memory). """ # Open the file @@ -155,8 +161,22 @@ class Raw(dict): self.fid = fid self.info = info - self._data = None - self._times = None + + if preload: + nchan = self.info['nchan'] + nsamp = self.last_samp - self.first_samp + 1 + if isinstance(preload, str): + # preload data using a memmap file + self._data = np.memmap(preload, mode='w+', dtype='float32', + shape=(nchan, nsamp)) + else: + self._data = np.empty((nchan, nsamp)) + + self._data, self._times = read_raw_segment(self, + data_buffer=self._data) + self._preloaded = True + else: + self._preloaded = False def __getitem__(self, item): """getting raw data content with python slicing""" @@ -187,7 +207,7 @@ class Raw(dict): if sel is not None and len(sel) == 0: raise Exception("Empty channel list") - if self._data is not None: + if self._preloaded: return (self._data[sel, start:stop], self._times[start:stop]) else: return read_raw_segment(self, start=start, stop=stop, sel=sel) @@ -197,6 +217,10 @@ class Raw(dict): def __setitem__(self, item, value): """setting raw data content with python slicing""" if isinstance(item, tuple): # slicing required + if not self._preloaded: + raise RuntimeError('Modifying data of Raw is only supported ' + 'when preloading is used. Use preload=True ' + '(or string) in the constructor.') if len(item) == 2: # channels and time instants time_slice = item[1] if isinstance(item[0], slice): @@ -223,25 +247,12 @@ class Raw(dict): if sel is not None and len(sel) == 0: raise Exception("Empty channel list") - if self._data is None: - # data needs to be loaded into memory - self.preload() - # set the data self._data[sel, start:stop] = value else: super(Raw, self).__setitem__(item, value) - def preload(self): - """preload the raw data into memory in order to modify the data and - for faster indexing - """ - if self._data is None: - data, times = self[:, :] - self._data = data - self._times = times - def save(self, fname, picks=None, tmin=None, tmax=None, buffer_size_sec=10, drop_small_buffer=False): """Save raw data to file @@ -332,7 +343,7 @@ class Raw(dict): return self.info['ch_names'] -def read_raw_segment(raw, start=0, stop=None, sel=None): +def read_raw_segment(raw, start=0, stop=None, sel=None, data_buffer=None): """Read a chunck of raw data Parameters @@ -351,8 +362,8 @@ def read_raw_segment(raw, start=0, stop=None, sel=None): sel: array, optional Indices of channels to select - node: tree node - The node of the tree where to look + data_buffer: array, optional + numpy array to fill with data read, must have the correct shape Returns ------- @@ -387,7 +398,13 @@ def read_raw_segment(raw, start=0, stop=None, sel=None): cal = np.diag(raw.cals.ravel()) if sel is None: - data = np.empty((nchan, stop - start)) + data_shape = (nchan, stop - start) + if data_buffer is not None: + if data_buffer.shape != data_shape: + raise ValueError('data_buffer has incorrect shape') + data = data_buffer + else: + data = np.empty(data_shape) if raw.proj is None and raw.comp is None: mult = None else: @@ -399,7 +416,13 @@ def read_raw_segment(raw, start=0, stop=None, sel=None): mult = raw.proj * raw.comp * cal else: - data = np.empty((len(sel), stop - start)) + data_shape = (len(sel), stop - start) + if data_buffer is not None: + if data_buffer.shape != data_shape: + raise ValueError('data_buffer has incorrect shape') + data = data_buffer + else: + data = np.empty(data_shape) if raw.proj is None and raw.comp is None: mult = None cal = np.diag(raw.cals[sel].ravel()) diff --git a/mne/fiff/tests/test_raw.py b/mne/fiff/tests/test_raw.py index 8952f70..abe4eb1 100644 --- a/mne/fiff/tests/test_raw.py +++ b/mne/fiff/tests/test_raw.py @@ -1,5 +1,6 @@ import os.path as op +import numpy as np from nose.tools import assert_true from numpy.testing import assert_array_almost_equal @@ -63,3 +64,32 @@ def test_io_raw(): assert_array_almost_equal(raw.info['dig'][0]['r'], raw2.info['dig'][0]['r']) fname = op.join(op.dirname(__file__), 'data', 'test_raw.fif') + + +def test_preload_modify(): + """ Test preloading and modifying data + """ + for preload in [False, True, 'memmap.dat']: + raw = Raw(fif_fname, preload=preload) + + nsamp = raw.last_samp - raw.first_samp + 1 + picks = pick_types(raw.info, meg='grad') + + data = np.random.randn(len(picks), nsamp / 2) + + try: + raw[picks, :nsamp / 2] = data + except RuntimeError as err: + if not preload: + continue + else: + raise err + + tmp_fname = 'raw.fif' + raw.save(tmp_fname) + + raw_new = Raw(tmp_fname) + data_new, _ = raw_new[picks, :nsamp / 2] + + assert_array_almost_equal(data, data_new) + diff --git a/mne/tests/test_raw.py b/mne/tests/test_raw.py deleted file mode 100644 index b43aaec..0000000 --- a/mne/tests/test_raw.py +++ /dev/null @@ -1,46 +0,0 @@ -import os.path as op - -import numpy as np -from numpy.testing import assert_array_almost_equal - -from ..datasets import sample -from ..fiff import pick_types - -from ..fiff.raw import Raw - -examples_folder = op.join(op.dirname(__file__), '..', '..', 'examples') -data_path = sample.data_path(examples_folder) -fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif') - - -def test_read_write(): - raw_orig = Raw(fname) - data_orig, times_orig = raw_orig[:, :] - - tmp_fname = 'tmp.fif' - raw_orig.save(tmp_fname) - - raw = Raw(tmp_fname) - data, times = raw[:, :] - - assert_array_almost_equal(data_orig, data) - assert_array_almost_equal(times_orig, times) - - -def test_modify_data(): - raw = Raw(fname) - - n_samp = raw.last_samp - raw.first_samp - picks = pick_types(raw.info, meg='grad') - - data = np.random.randn(len(picks), n_samp / 2) - - raw[picks, :n_samp / 2] = data - - tmp_fname = 'tmp.fif' - raw.save(tmp_fname) - - raw_new = Raw(tmp_fname) - data_new, _ = raw_new[picks, :n_samp / 2] - - assert_array_almost_equal(data, data_new) \ No newline at end of file -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/python-mne.git _______________________________________________ debian-med-commit mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/debian-med-commit
