Repository: climate Updated Branches: refs/heads/master 3d0c32116 -> b08d37940
CLIMATE-564 - Managing multiple netcdf files stored on a local machine - A new file loader load_multiple_files can read netcdf files with common file name patterns. Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/579c1f13 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/579c1f13 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/579c1f13 Branch: refs/heads/master Commit: 579c1f1384583abf83b4cb9c2b8e4eeb6c26bd19 Parents: d4eeb03 Author: huikyole <[email protected]> Authored: Wed Aug 12 01:34:44 2015 -0700 Committer: huikyole <[email protected]> Committed: Wed Aug 12 01:34:44 2015 -0700 ---------------------------------------------------------------------- ocw/data_source/local.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/579c1f13/ocw/data_source/local.py ---------------------------------------------------------------------- diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py index 474dffb..be33998 100644 --- a/ocw/data_source/local.py +++ b/ocw/data_source/local.py @@ -268,3 +268,38 @@ def load_file(file_path, return Dataset(lats, lons, times, values, variable=variable_name, units=variable_unit, name=name, origin=origin) + +def load_multiple_files(data_info): + ''' load files from multiple datasets and return an array of OCW datasets + + :param data_path: ['datasets']['targets'] in a configuration yaml file. + :type data_path: :class:`list` + + :returns: An array of OCW Dataset objects, an array of dataset names + :rtype: :class:`list` + ''' + + data_filenames = glob(data_info['path']) + data_filenames.sort() + # number of files + ndata = len(data_filenames) + if ndata == 1: + try: + data_name = [data_info['data_name']] + except: + data_name =['ref'] + else: + data_name = [] + data_filenames_reversed = [] + for element in data_filenames: + data_filenames_reversed.append(element[::-1]) + prefix = os.path.commonprefix(data_filenames) + postfix = os.path.commonprefix(data_filenames_reversed)[::-1] + for element in data_filenames: + data_name.append(element.replace(prefix,'').replace(postfix,'')) + + datasets = [] + for filename in data_filenames: + datasets.append(load_file(filename, data_info['variable'])) + + return datasets, data_name
