Repository: climate Updated Branches: refs/heads/master 89a96c0fe -> 6ab02d8f6
CLIMATE-633 - Add support for loading WRF data 2D formatted files - Add load_WRF_2d_files to datasource.local for handling the Weather Research and Forecasting 2D field datasets. - Update normalize_lat_lon_values to handle multi-dim lat/lon fields. Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/13692480 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/13692480 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/13692480 Branch: refs/heads/master Commit: 136924803f05841e96cad894eb8f073fecee648a Parents: 89a96c0 Author: Huikyo Lee <[email protected]> Authored: Tue Apr 28 15:39:46 2015 -0400 Committer: Michael Joyce <[email protected]> Committed: Mon May 11 08:13:51 2015 -0700 ---------------------------------------------------------------------- ocw/data_source/local.py | 46 ++++++++++++++++++++++++++ ocw/dataset.py | 9 +++--- ocw/utils.py | 75 ++++++++++++++++++++++--------------------- 3 files changed, 89 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/13692480/ocw/data_source/local.py ---------------------------------------------------------------------- diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py index 2de03f5..474dffb 100644 --- a/ocw/data_source/local.py +++ b/ocw/data_source/local.py @@ -17,6 +17,8 @@ import calendar from datetime import timedelta ,datetime +from time import strptime +from glob import glob import re import string @@ -109,6 +111,50 @@ def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var): ) raise ValueError(error) +def load_WRF_2d_files(file_path, + filename_pattern, + variable_name, + name=''): + ''' Load multiple WRF (or nuWRF) original output files containing 2D fields such as precipitation and surface variables into a Dataset. + The dataset can be spatially subset. + :param file_path: Directory to the NetCDF file to load. + :type file_path: :mod:`string` + :param filename_pattern: Path to the NetCDF file to load. + :type filename_pattern: :list:`string` + :param variable_name: The variable name to load from the NetCDF file. + :type variable_name: :mod:`string` + :param name: (Optional) A name for the loaded dataset. + :type name: :mod:`string` + :returns: An OCW Dataset object with the requested variable's data from + the NetCDF file. + :rtype: :class:`dataset.Dataset` + :raises ValueError: + ''' + + WRF_files = [] + for pattern in filename_pattern: + WRF_files.extend(glob(file_path + pattern)) + WRF_files.sort() + + file_object_first = netCDF4.Dataset(WRF_files[0]) + lats = file_object_first.variables['XLAT'][0,:] + lons = file_object_first.variables['XLONG'][0,:] + + times = [] + for ifile, file in enumerate(WRF_files): + file_object = netCDF4.Dataset(file) + time_struct_parsed = strptime(file[-19:],"%Y-%m-%d_%H:%M:%S") + for ihour in numpy.arange(24): + times.append(datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour)) + values0= file_object.variables[variable_name][:] + if ifile == 0: + values = file_object.variables[variable_name][:] + else: + values = numpy.concatenate((values, file_object.variables[variable_name][:])) + file_object.close() + times = numpy.array(times) + return Dataset(lats, lons, times, values, variable_name, name=name) + def load_file(file_path, variable_name, variable_unit = None, http://git-wip-us.apache.org/repos/asf/climate/blob/13692480/ocw/dataset.py ---------------------------------------------------------------------- diff --git a/ocw/dataset.py b/ocw/dataset.py index d231f0e..d33bdd4 100644 --- a/ocw/dataset.py +++ b/ocw/dataset.py @@ -172,13 +172,12 @@ class Dataset: value_dim = len(values.shape) lat_count = lats.shape[0] lon_count = lons.shape[0] + + if lat_dim == 2 and lon_dim == 2: + lon_count = lons.shape[1] time_count = times.shape[0] - if lat_dim != 1: - err_msg = "Latitude Array should be 1 dimensional. %s dimensions found." % lat_dim - elif lon_dim != 1: - err_msg = "Longitude Array should be 1 dimensional. %s dimensions found." % lon_dim - elif time_dim != 1: + if time_dim != 1: err_msg = "Time Array should be 1 dimensional. %s dimensions found." % time_dim elif value_dim < 2: err_msg = "Value Array should be at least 2 dimensional. %s dimensions found." % value_dim http://git-wip-us.apache.org/repos/asf/climate/blob/13692480/ocw/utils.py ---------------------------------------------------------------------- diff --git a/ocw/utils.py b/ocw/utils.py index f367dab..e8a004c 100644 --- a/ocw/utils.py +++ b/ocw/utils.py @@ -179,42 +179,45 @@ def normalize_lat_lon_values(lats, lons, values): :raises ValueError: If the lat/lon values are not sorted. ''' - # Avoid unnecessary shifting if all lons are higher than 180 - if lons.min() > 180: - lons -= 360 - - # Make sure lats and lons are monotonically increasing - lats_decreasing = np.diff(lats) < 0 - lons_decreasing = np.diff(lons) < 0 - - # If all values are decreasing then they just need to be reversed - lats_reversed, lons_reversed = lats_decreasing.all(), lons_decreasing.all() - - # If the lat values are unsorted then raise an exception - if not lats_reversed and lats_decreasing.any(): - raise ValueError('Latitudes must be sorted.') - - # Perform same checks now for lons - if not lons_reversed and lons_decreasing.any(): - raise ValueError('Longitudes must be sorted.') - - # Also check if lons go from [0, 360), and convert to [-180, 180) - # if necessary - lons_shifted = lons.max() > 180 - lats_out, lons_out, data_out = lats[:], lons[:], values[:] - # Now correct data if latlon grid needs to be shifted - if lats_reversed: - lats_out = lats_out[::-1] - data_out = data_out[..., ::-1, :] - - if lons_reversed: - lons_out = lons_out[::-1] - data_out = data_out[..., ::-1] - - if lons_shifted: - data_out, lons_out = shiftgrid(180, data_out, lons_out, start=False) - - return lats_out, lons_out, data_out + if lats.ndim ==1 and lons.ndim ==1: + # Avoid unnecessary shifting if all lons are higher than 180 + if lons.min() > 180: + lons -= 360 + + # Make sure lats and lons are monotonically increasing + lats_decreasing = np.diff(lats) < 0 + lons_decreasing = np.diff(lons) < 0 + + # If all values are decreasing then they just need to be reversed + lats_reversed, lons_reversed = lats_decreasing.all(), lons_decreasing.all() + + # If the lat values are unsorted then raise an exception + if not lats_reversed and lats_decreasing.any(): + raise ValueError('Latitudes must be sorted.') + + # Perform same checks now for lons + if not lons_reversed and lons_decreasing.any(): + raise ValueError('Longitudes must be sorted.') + + # Also check if lons go from [0, 360), and convert to [-180, 180) + # if necessary + lons_shifted = lons.max() > 180 + lats_out, lons_out, data_out = lats[:], lons[:], values[:] + # Now correct data if latlon grid needs to be shifted + if lats_reversed: + lats_out = lats_out[::-1] + data_out = data_out[..., ::-1, :] + + if lons_reversed: + lons_out = lons_out[::-1] + data_out = data_out[..., ::-1] + + if lons_shifted: + data_out, lons_out = shiftgrid(180, data_out, lons_out, start=False) + + return lats_out, lons_out, data_out + else: + return lats, lons, values def reshape_monthly_to_annually(dataset):
