debug
Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/cefd9d63 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/cefd9d63 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/cefd9d63 Branch: refs/heads/master Commit: cefd9d632017002f1c6abb0f650e3fb8416f0eab Parents: 69acb0e Author: Huikyo Lee <[email protected]> Authored: Fri Mar 20 00:41:33 2015 -0700 Committer: Huikyo Lee <[email protected]> Committed: Fri Mar 20 00:41:33 2015 -0700 ---------------------------------------------------------------------- ocw/data_source/local.py | 178 ------------------------------------------ ocw/dataset.py | 22 +++--- ocw/utils.py | 4 - 3 files changed, 11 insertions(+), 193 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/data_source/local.py ---------------------------------------------------------------------- diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py index 3fe6497..2de03f5 100644 --- a/ocw/data_source/local.py +++ b/ocw/data_source/local.py @@ -26,7 +26,6 @@ import ocw.utils as utils import netCDF4 import numpy import numpy.ma as ma -import glob LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes'] LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes'] @@ -212,182 +211,6 @@ def load_file(file_path, else: values = values [:,:,:,elevation_index] -<<<<<<< HEAD - return Dataset(lats, lons, times, values, variable_name, name=name) - -def load_files(file_path, - filename_pattern, - variable_name, - elevation_index=0, - name='', - lat_name=None, - lon_name=None, - time_name=None, - latitude_range=None, - longitude_range=None): - ''' Load multiple NetCDF files whose file names have common patterns into a Dataset. - The dataset can be spatially subset. - - :param file_path: Directory to the NetCDF file to load. - :type file_path: :mod:`string` - - :param filename_pattern: Path to the NetCDF file to load. - :type filename_pattern: :list:`string` - - :param variable_name: The variable name to load from the NetCDF file. - :type variable_name: :mod:`string` - - :param elevation_index: (Optional) The elevation index for which data should - be returned. Climate data is often times 4 dimensional data. Some - datasets will have readins at different height/elevation levels. OCW - expects 3D data so a single layer needs to be stripped out when loading. - By default, the first elevation layer is used. If desired you may - specify the elevation value to use. - :type elevation_index: :class:`int` - - :param name: (Optional) A name for the loaded dataset. - :type name: :mod:`string` - - :param lat_name: (Optional) The latitude variable name to extract from the - dataset. - :type lat_name: :mod:`string` - - :param lon_name: (Optional) The longitude variable name to extract from the - dataset. - :type lon_name: :mod:`string` - - :param time_name: (Optional) The time variable name to extract from the - dataset. - :type time_name: :mod:`string` - - :param latitude_range: (Optional) southern and northern boundary of the sub-region - :type latitude_range: :list:float - - :param longitude_range: (Optional) western and eastern boundary of the sub-region - :type longitude_range: :list:float - - :returns: An OCW Dataset object with the requested variable's data from - the NetCDF file. - :rtype: :class:`dataset.Dataset` - - :raises ValueError: When the specified file path cannot be loaded by ndfCDF4 - or when the lat/lon/time variable name cannot be determined - automatically. - ''' - - netcdf_files= [] - for pattern in filename_pattern: - netcdf_files.extend(glob.glob(file_path+pattern)) - netcdf_files.sort() - - try: - netcdf = netCDF4.Dataset(netcdf_files[0], mode='r') - except RuntimeError: - err = "Dataset filepath is invalid. Please ensure it is correct." - raise ValueError(err) - except: - err = ( - "The given file cannot be loaded. Please ensure that it is a valid " - "NetCDF file. If problems persist, report them to the project's " - "mailing list." - ) - raise ValueError(err) - - if not lat_name: - lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name) - if not lon_name: - lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name) - if not time_name: - time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name) - - lats = netcdf.variables[lat_name][:] - lons = netcdf.variables[lon_name][:] - - if latitude_range and longitude_range: - if lats.ndim == 1: - x_index = numpy.where((lons>=numpy.min(longitude_range)) & (lons<=numpy.max(longitude_range)))[0] - y_index = numpy.where((lats>=numpy.min(latitude_range)) & (lats<=numpy.max(latitude_range)))[0] - lats = lats[y_index] - lons = lons[x_index] - else: - y_index,x_index = numpy.where((lons>=numpy.min(longitude_range)) & (lons<=numpy.max(longitude_range)) & (lats>=numpy.min(latitude_range)) & (lats<=numpy.max(latitude_range))) - lats = lats[y_index, x_index] - lons = lons[y_index, x_index] - else: - y_index = np.arange(lats.shape[0]) - x_index = np.arange(lons.shape[-1]) - - time_raw_values = netcdf.variables[time_name] - for attr, value in time_raw_values.__dict__.iteritems(): - if 'unit' in attr.lower(): - time_unit = value - times = netCDF4.num2date(time_raw_values[:], units = time_unit) - times = numpy.array(times) - - # check the variable structure before reading data from the open file - variable = netcdf.variables[variable_name] - # If the values are 4D then we need to strip out the elevation index - if len(variable.shape) == 4: - # Determine the set of possible elevation dimension names excluding - # the list of names that are used for the lat, lon, and time values. - dims = netcdf.variables[variable_name].dimensions - dimension_names = [dim_name.encode() for dim_name in dims] - lat_lon_time_var_names = [lat_name, lon_name, time_name] - - elev_names = set(dimension_names) - set(lat_lon_time_var_names) - - # Grab the index value for the elevation values - level_index = dimension_names.index(elev_names.pop()) - - # Strip out the elevation values so we're left with a 3D array. - if level_index == 0: - values = variable[elevation_index,:,y_index,x_index] - elif level_index == 1: - values = variable[:,elevation_index,y_index,x_index] - else: - raise ValueError('The structure of this variable does not follow the community standard') - if len(netcdf_files) >1: - for netcdf_file in netcdf_files[1:]: - netcdf.close() - netcdf = netCDF4.Dataset(netcdf_file, mode='r') - time_raw_values = netcdf.variables[time_name] - for attr, value in time_raw_values.__dict__.iteritems(): - if 'unit' in attr.lower(): - time_unit = value - times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units = time_unit)) - if level_index == 0: - values = numpy.concatenate((values, netcdf.variables[variable_name][elevation_index,:,y_index,x_index]), axis=0) - elif level_index == 1: - values = numpy.concatenate((values, netcdf.variables[variable_name][:,elevation_index,y_index,x_index]), axis=0) - - elif len(variable.shape) == 3: - values = variable[:,y_index,x_index] - - if len(netcdf_files) >1: - for netcdf_file in netcdf_files[1:]: - netcdf.close() - netcdf = netCDF4.Dataset(netcdf_file, mode='r') - time_raw_values = netcdf.variables[time_name] - for attr, value in time_raw_values.__dict__.iteritems(): - if 'unit' in attr.lower(): - time_unit = value - times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units=time_unit)) - values = numpy.concatenate((values, netcdf.variables[variable_name][:,y_index,x_index]), axis=0) - elif len(variable.shape) == 2: - values = (variable[y_index,x_index]).reshape((1,y_index.size,x_index.size)) - if len(netcdf_files) >1: - for netcdf_file in netcdf_files[1:]: - netcdf.close() - netcdf = netCDF4.Dataset(netcdf_file, mode='r') - time_raw_values = netcdf.variables[time_name] - for attr, value in time_raw_values.__dict__.iteritems(): - if 'unit' in attr.lower(): - time_unit = value - times = numpy.append(times, netCDF4.num2date(time_raw_values[:], units=time_unit)) - values = numpy.concatenate((values, (netcdf.variables[variable_name][y_index,x_index]).reshape((1,y_index.size,x_index.size))), axis=0) - return Dataset(lats, lons, times, values, variable_name, name=name) - -======= origin = { 'source': 'local', 'path': file_path, @@ -399,4 +222,3 @@ def load_files(file_path, return Dataset(lats, lons, times, values, variable=variable_name, units=variable_unit, name=name, origin=origin) ->>>>>>> upstream/master http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/dataset.py ---------------------------------------------------------------------- diff --git a/ocw/dataset.py b/ocw/dataset.py index 48e7a22..ee86532 100644 --- a/ocw/dataset.py +++ b/ocw/dataset.py @@ -69,7 +69,7 @@ class Dataset: :raises: ValueError ''' self._validate_inputs(lats, lons, times, values) - #lats, lons, values = utils.normalize_lat_lon_values(lats, lons, values) + lats, lons, values = utils.normalize_lat_lon_values(lats, lons, values) self.lats = lats self.lons = lons @@ -180,17 +180,17 @@ class Dataset: err_msg = "Longitude Array should be 1 dimensional. %s dimensions found." % lon_dim elif time_dim != 1: err_msg = "Time Array should be 1 dimensional. %s dimensions found." % time_dim - #elif value_dim != 3: - # err_msg = "Value Array should be 3 dimensional. %s dimensions found." % value_dim + elif value_dim != 3: + err_msg = "Value Array should be 3 dimensional. %s dimensions found." % value_dim # Finally check that the Values array conforms to the proper shape - #elif values.shape != (time_count, lat_count, lon_count): - # err_msg = """Value Array must be of shape (times, lats, lons). -#Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count, -# lat_count, -# lon_count, -# values.shape[0], -# values.shape[1], -# values.shape[2]) + elif values.shape != (time_count, lat_count, lon_count): + err_msg = """Value Array must be of shape (times, lats, lons). +Expected shape (%s, %s, %s) but received (%s, %s, %s)""" % (time_count, + lat_count, + lon_count, + values.shape[0], + values.shape[1], + values.shape[2]) if err_msg: logger.error(err_msg) raise ValueError(err_msg) http://git-wip-us.apache.org/repos/asf/climate/blob/cefd9d63/ocw/utils.py ---------------------------------------------------------------------- diff --git a/ocw/utils.py b/ocw/utils.py index e3d3311..e778d40 100755 --- a/ocw/utils.py +++ b/ocw/utils.py @@ -289,7 +289,6 @@ def calc_climatology_year(dataset): return annually_mean, total_mean -<<<<<<< HEAD def calc_climatology_season(month_start, month_end, dataset): ''' Calculate seasonal mean and time series for given months. @@ -345,6 +344,3 @@ def calc_climatology_monthly(dataset): raise ValueError(error) else: return reshape_monthly_to_annually(dataset).mean(axis=0) - -======= ->>>>>>> cfb120e5eaea004e850b884e49d41f8a0c269b75
