Repository: climate Updated Branches: refs/heads/master f931bb12c -> 18fe8a385
http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_dataset.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_dataset.py b/ocw/tests/test_dataset.py index 9d77400..a51cddc 100644 --- a/ocw/tests/test_dataset.py +++ b/ocw/tests/test_dataset.py @@ -24,6 +24,7 @@ import datetime as dt class TestDatasetAttributes(unittest.TestCase): + def setUp(self): self.lat = np.array([10, 12, 14, 16, 18]) self.lon = np.array([100, 102, 104, 106, 108]) @@ -64,6 +65,7 @@ class TestDatasetAttributes(unittest.TestCase): class TestInvalidDatasetInit(unittest.TestCase): + def setUp(self): self.lat = np.array([10, 12, 14, 16, 18]) self.lon = np.array([100, 102, 104, 106, 108]) @@ -122,6 +124,7 @@ class TestInvalidDatasetInit(unittest.TestCase): class TestDatasetFunctions(unittest.TestCase): + def setUp(self): self.lat = np.array([10, 12, 14, 16, 18]) self.lon = np.array([100, 102, 104, 106, 108]) @@ -213,14 +216,18 @@ class TestDatasetFunctions(unittest.TestCase): class TestBounds(unittest.TestCase): + def setUp(self): self.bounds_rectangular = Bounds(lat_min=-80, lat_max=80, # Lats - lon_min=-160, lon_max=160, # Lons - start=dt.datetime(2000, 1, 1), # Start time - end=dt.datetime(2002, 1, 1)) # End time + lon_min=-160, lon_max=160, # Lons + start=dt.datetime( + 2000, 1, 1), # Start time + end=dt.datetime(2002, 1, 1)) # End time self.bounds_CORDEX = Bounds(boundary_type='CORDEX South Asia') - self.bounds_us_states = Bounds(boundary_type='us_states', us_states=['CA','NV','AZ']) - self.bounds_countries = Bounds(boundary_type='countries', countries=['United States','Canada','Mexico']) + self.bounds_us_states = Bounds( + boundary_type='us_states', us_states=['CA', 'NV', 'AZ']) + self.bounds_countries = Bounds(boundary_type='countries', countries=[ + 'United States', 'Canada', 'Mexico']) def test_keywords(self): self.assertEqual(self.bounds_rectangular.boundary_type, 'rectangular') @@ -228,17 +235,18 @@ class TestBounds(unittest.TestCase): self.assertEqual(self.bounds_rectangular.lat_max, 80) self.assertEqual(self.bounds_rectangular.lon_min, -160) self.assertEqual(self.bounds_rectangular.lon_max, 160) - self.assertEqual(self.bounds_rectangular.start, dt.datetime(2000,1,1)) - self.assertEqual(self.bounds_rectangular.end, dt.datetime(2002,1,1)) + self.assertEqual(self.bounds_rectangular.start, + dt.datetime(2000, 1, 1)) + self.assertEqual(self.bounds_rectangular.end, dt.datetime(2002, 1, 1)) self.assertEqual(self.bounds_CORDEX.boundary_type, 'CORDEX South Asia') self.assertEqual(self.bounds_CORDEX.lat_min, -15.23) self.assertEqual(self.bounds_CORDEX.lat_max, 45.07) self.assertEqual(self.bounds_CORDEX.lon_min, 19.88) self.assertEqual(self.bounds_CORDEX.lon_max, 115.55) - + self.assertEqual(self.bounds_us_states.boundary_type, 'us_states') - + self.assertEqual(self.bounds_countries.boundary_type, 'countries') # Temporal tests http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_dataset_loader.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_dataset_loader.py b/ocw/tests/test_dataset_loader.py index b3c613b..5331310 100644 --- a/ocw/tests/test_dataset_loader.py +++ b/ocw/tests/test_dataset_loader.py @@ -22,7 +22,9 @@ import numpy as np from ocw.dataset import Dataset from ocw.dataset_loader import DatasetLoader + class TestDatasetLoader(unittest.TestCase): + def setUp(self): # Read netCDF file self.file_path = create_netcdf_object() @@ -88,6 +90,7 @@ class TestDatasetLoader(unittest.TestCase): np.testing.assert_array_equal(self.loader.datasets[1].values, self.values2) + def build_dataset(*args, **kwargs): ''' Wrapper to Dataset constructor from fictitious 'foo' data_source. @@ -95,6 +98,7 @@ def build_dataset(*args, **kwargs): origin = {'source': 'foo'} return Dataset(*args, origin=origin, **kwargs) + def create_netcdf_object(): # To create the temporary netCDF file file_path = '/tmp/temporaryNetcdf.nc' @@ -138,7 +142,8 @@ def create_netcdf_object(): values[:] = values_data # Assign time info to time variable netCDF_file.variables['time'].units = 'months since 2001-01-01 00:00:00' - netCDF_file.variables['alt_time'].units = 'months since 2001-04-01 00:00:00' + netCDF_file.variables[ + 'alt_time'].units = 'months since 2001-04-01 00:00:00' netCDF_file.variables['value'].units = 'foo_units' netCDF_file.close() return file_path http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_dataset_processor.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_dataset_processor.py b/ocw/tests/test_dataset_processor.py index 42bdc2c..9538e91 100644 --- a/ocw/tests/test_dataset_processor.py +++ b/ocw/tests/test_dataset_processor.py @@ -30,6 +30,7 @@ logging.basicConfig(level=logging.CRITICAL) class TestTemporalSubset(unittest.TestCase): + def setUp(self): self.ten_year_dataset = ten_year_monthly_dataset() @@ -92,6 +93,7 @@ class TestTemporalSubset(unittest.TestCase): class TestTemporalRebinWithTimeIndex(unittest.TestCase): + def setUp(self): self.ten_year_dataset = ten_year_monthly_dataset() @@ -137,6 +139,7 @@ class TestTemporalRebinWithTimeIndex(unittest.TestCase): class TestVariableUnitConversion(unittest.TestCase): + def setUp(self): self.ten_year_dataset = ten_year_monthly_dataset() self.ten_year_dataset.variable = 'temp' @@ -199,6 +202,7 @@ class TestVariableUnitConversion(unittest.TestCase): class TestTemporalSlice(unittest.TestCase): + def test_returned_dataset_times(self): ''' Tests returned dataset times values ''' self.ten_year_dataset = ten_year_monthly_dataset() @@ -224,6 +228,7 @@ class TestTemporalSlice(unittest.TestCase): class TestEnsemble(unittest.TestCase): + def test_unequal_dataset_shapes(self): self.ten_year_dataset = ten_year_monthly_dataset() self.two_year_dataset = two_year_daily_dataset() @@ -396,6 +401,7 @@ class TestSpatialRegrid(unittest.TestCase): class TestNormalizeDatasetDatetimes(unittest.TestCase): + def setUp(self): self.monthly_dataset = ten_year_monthly_15th_dataset() self.daily_dataset = two_year_daily_2hr_dataset() @@ -426,6 +432,7 @@ class TestNormalizeDatasetDatetimes(unittest.TestCase): class TestSubset(unittest.TestCase): + def setUp(self): self.target_dataset = ten_year_monthly_dataset() self.name = 'foo' @@ -508,6 +515,7 @@ class TestSubset(unittest.TestCase): class TestSafeSubset(unittest.TestCase): + def setUp(self): lats = np.array(range(-60, 61, 1)) lons = np.array(range(-170, 171, 1)) @@ -579,6 +587,7 @@ class TestSafeSubset(unittest.TestCase): class TestFailingSubset(unittest.TestCase): + def setUp(self): self.target_dataset = ten_year_monthly_dataset() self.target_dataset.lats = np.array(range(-89, 88, 2)) @@ -623,6 +632,7 @@ class TestFailingSubset(unittest.TestCase): class TestNetCDFWrite(unittest.TestCase): + def setUp(self): self.ds = ten_year_monthly_dataset() self.ds_2d = ten_year_monthly_dataset(latlon2d=True) http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_evaluation.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_evaluation.py b/ocw/tests/test_evaluation.py index 9053e52..2bb10cc 100644 --- a/ocw/tests/test_evaluation.py +++ b/ocw/tests/test_evaluation.py @@ -26,6 +26,7 @@ from ocw.metrics import Bias, TemporalStdDev class TestEvaluation(unittest.TestCase): + def setUp(self): self.eval = Evaluation(None, [], []) http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_local.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_local.py b/ocw/tests/test_local.py index 919a6c4..645f486 100644 --- a/ocw/tests/test_local.py +++ b/ocw/tests/test_local.py @@ -26,6 +26,7 @@ import ocw.data_source.local as local class test_load_file(unittest.TestCase): + def setUp(self): # Read netCDF file self.file_path = create_netcdf_object() @@ -102,6 +103,7 @@ class test_load_file(unittest.TestCase): class TestLoadMultipleFiles(unittest.TestCase): + def setUp(self): # Read netCDF file self.file_path = create_netcdf_object() @@ -143,8 +145,8 @@ class TestLoadMultipleFiles(unittest.TestCase): def test_load_multiple_files_custom_dataset_name(self): """Test adding a custom name to a dataset""" dataset = local.load_multiple_files(self.file_path, - "value", - dataset_name='foo') + "value", + dataset_name='foo') self.assertEqual(dataset[0].name, 'foo') def test_dataset_origin(self): @@ -156,6 +158,7 @@ class TestLoadMultipleFiles(unittest.TestCase): class TestLoadDatasetFromMultipleNetcdfFiles(unittest.TestCase): + def setUp(self): self.file_path = create_netcdf_object() self.netCDF_file = netCDF4.Dataset(self.file_path, 'r+') @@ -315,7 +318,8 @@ def create_netcdf_object(): values[:] = values_data # Assign time info to time variable netCDF_file.variables['time'].units = 'months since 2001-01-01 00:00:00' - netCDF_file.variables['alt_time'].units = 'months since 2001-04-01 00:00:00' + netCDF_file.variables[ + 'alt_time'].units = 'months since 2001-04-01 00:00:00' netCDF_file.variables['value'].units = 'foo_units' netCDF_file.close() return file_path http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_plotter.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_plotter.py b/ocw/tests/test_plotter.py index f824670..9020fd2 100644 --- a/ocw/tests/test_plotter.py +++ b/ocw/tests/test_plotter.py @@ -23,6 +23,7 @@ from ocw import plotter class TestNiceIntervalsFunction(unittest.TestCase): + def test_nice_intervals(self): test_array = np.arange(0, 30) expected_array = np.arange(0, 30, 3)[1::] @@ -46,6 +47,7 @@ class TestNiceIntervalsFunction(unittest.TestCase): class TestBestGridShapeFunction(unittest.TestCase): + def test_returned_shape_small(self): nplots = 2 oldshape = (2, 2) @@ -75,6 +77,7 @@ class TestBestGridShapeFunction(unittest.TestCase): class TestFigshapeFunction(unittest.TestCase): + def test_small_gridshape_size(self): gridshape = (2, 2) expected_width = 8.5 http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_rcmed.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_rcmed.py b/ocw/tests/test_rcmed.py index c9cebe2..35975a1 100644 --- a/ocw/tests/test_rcmed.py +++ b/ocw/tests/test_rcmed.py @@ -27,6 +27,7 @@ import ocw.data_source.rcmed as rcmed class CustomAssertions: # Custom Assertions to handle Numpy Arrays + def assert1DArraysEqual(self, array1, array2): self.assertSequenceEqual(tuple(array1), tuple(array2)) http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/tests/test_utils.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_utils.py b/ocw/tests/test_utils.py index d35bc55..987f2e6 100644 --- a/ocw/tests/test_utils.py +++ b/ocw/tests/test_utils.py @@ -65,6 +65,7 @@ class TestDecodeTimes(unittest.TestCase): class TestTimeUnitsParse(unittest.TestCase): + def test_valid_parse(self): units = utils.parse_time_units('minutes since a made up date') @@ -79,6 +80,7 @@ class TestTimeUnitsParse(unittest.TestCase): class TestTimeBaseParse(unittest.TestCase): + def test_valid_time_base(self): base_time = utils.parse_time_base('days since 1988-06-10 00:00:00') start_time = datetime.datetime.strptime( @@ -95,6 +97,7 @@ class TestTimeBaseParse(unittest.TestCase): class TestBaseTimeStringParse(unittest.TestCase): + def test_valid_time_base_string_parse(self): base = utils.parse_base_time_string('days since 1988-06-10 00:00:00') @@ -109,6 +112,7 @@ class TestBaseTimeStringParse(unittest.TestCase): class TestNormalizeLatLonValues(unittest.TestCase): + def setUp(self): times = np.array([datetime.datetime(2000, x, 1) for x in range(1, 13)]) self.lats = np.arange(-30, 30) @@ -203,6 +207,7 @@ class TestNormalizeLatLonValues(unittest.TestCase): class TestGetTemporalOverlap(unittest.TestCase): + def setUp(self): self.lat = np.array([10, 12, 14, 16, 18]) self.lon = np.array([100, 102, 104, 106, 108]) @@ -265,6 +270,7 @@ class TestReshapeMonthlyToAnnually(unittest.TestCase): class TestCalcTemporalMean(unittest.TestCase): + def setUp(self): self.lat = np.array([10, 12, 14]) self.lon = np.array([100, 102, 104]) @@ -286,6 +292,7 @@ class TestCalcTemporalMean(unittest.TestCase): class TestCalcAreaWeightedSpatialAverage(unittest.TestCase): + def setUp(self): self.lat = np.array([10, 12, 14]) self.lon = np.array([100, 102, 104]) http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/ocw/utils.py ---------------------------------------------------------------------- diff --git a/ocw/utils.py b/ocw/utils.py index 5226509..4876966 100755 --- a/ocw/utils.py +++ b/ocw/utils.py @@ -385,13 +385,16 @@ def get_temporal_overlap(dataset_array): def calc_subregion_area_mean_and_std(dataset_array, subregions): - ''' Calculate area mean and standard deviation values for a given + ''' Calculate area mean and standard deviation values for a given \ subregions using datasets on common grid points - :param dataset_array: An array of OCW Dataset Objects - :type list: - :param subregions: list of subregions + + :param dataset_array: An array of OCW Dataset Objects \ + :type list: :mod:'list' + + :param subregions: list of subregions \ :type subregions: :class:`numpy.ma.array` - :returns: area averaged time series for the dataset of shape + + :returns: area averaged time series for the dataset of shape \ (ntime, nsubregion) ''' @@ -450,6 +453,7 @@ def calc_area_weighted_spatial_average(dataset, area_weight=False): return spatial_average + def shapefile_boundary(boundary_type, region_names): ''' :param boundary_type: The type of spatial subset boundary @@ -472,48 +476,50 @@ def shapefile_boundary(boundary_type, region_names): elif boundary_type == 'countries': for region_name in region_names: for iregion, region_info in enumerate(map_read.countries_info): - if region_info['COUNTRY'].replace(" ","").lower() == region_name.replace(" ","").lower(): + if region_info['COUNTRY'].replace(" ", "").lower() == region_name.replace(" ", "").lower(): regions.append(np.array(map_read.countries[iregion])) return regions + def CORDEX_boundary(domain_name): ''' :param domain_name: CORDEX domain name (http://www.cordex.org/) :type domain_name: :mod:'string' ''' - if domain_name =='southamerica': - return -57.61, 18.50, 254.28-360., 343.02-360. - elif domain_name =='centralamerica': - return -19.46, 34.83, 235.74-360., 337.78-360. - elif domain_name =='northamerica': - return 12.55, 75.88, 189.26-360., 336.74-360. - elif domain_name =='europe': - return 22.20, 71.84, 338.23-360., 64.4 - elif domain_name =='africa': - return -45.76, 42.24, 335.36-360., 60.28 - elif domain_name =='southasia': + if domain_name == 'southamerica': + return -57.61, 18.50, 254.28 - 360., 343.02 - 360. + elif domain_name == 'centralamerica': + return -19.46, 34.83, 235.74 - 360., 337.78 - 360. + elif domain_name == 'northamerica': + return 12.55, 75.88, 189.26 - 360., 336.74 - 360. + elif domain_name == 'europe': + return 22.20, 71.84, 338.23 - 360., 64.4 + elif domain_name == 'africa': + return -45.76, 42.24, 335.36 - 360., 60.28 + elif domain_name == 'southasia': return -15.23, 45.07, 19.88, 115.55 - elif domain_name =='eastasia': - return -0.10, 61.90, 51.59, 179.99 - elif domain_name =='centralasia': - return 18.34, 69.37, 11.05, 139.13 - elif domain_name =='australasia': + elif domain_name == 'eastasia': + return -0.10, 61.90, 51.59, 179.99 + elif domain_name == 'centralasia': + return 18.34, 69.37, 11.05, 139.13 + elif domain_name == 'australasia': return -52.36, 12.21, 89.25, 179.99 - elif domain_name =='antartica': - return -89.48,-56.00, -179.00, 179.00 - elif domain_name =='artic': + elif domain_name == 'antartica': + return -89.48, -56.00, -179.00, 179.00 + elif domain_name == 'artic': return 46.06, 89.50, -179.00, 179.00 - elif domain_name =='mediterranean': - return 25.63, 56.66, 339.79-360.00, 50.85 - elif domain_name =='middleeastnorthafrica': - return -7.00, 45.00, 333.00-360.00, 76.00 - elif domain_name =='southeastasia': - return -15.14, 27.26, 89.26, 146.96 + elif domain_name == 'mediterranean': + return 25.63, 56.66, 339.79 - 360.00, 50.85 + elif domain_name == 'middleeastnorthafrica': + return -7.00, 45.00, 333.00 - 360.00, 76.00 + elif domain_name == 'southeastasia': + return -15.14, 27.26, 89.26, 146.96 else: err = "Invalid CORDEX domain name" raise ValueError(err) -def mask_using_shapefile_info(lons, lats, masked_regions, extract = True): + +def mask_using_shapefile_info(lons, lats, masked_regions, extract=True): if lons.ndim == 2 and lats.ndim == 2: lons_2d = lons lats_2d = lats @@ -527,21 +533,23 @@ def mask_using_shapefile_info(lons, lats, masked_regions, extract = True): if iregion == 0: mask = mask0 else: - mask = mask | mask0 + mask = mask | mask0 if extract: mask = np.invert(mask) return mask -def regrid_spatial_mask(target_lon, target_lat, mask_lon, mask_lat, mask_var, + +def regrid_spatial_mask(target_lon, target_lat, mask_lon, mask_lat, mask_var, user_mask_values, extract=True): target_lons, target_lats = convert_lat_lon_2d_array(target_lon, target_lat) mask_lons, mask_lats = convert_lat_lon_2d_array(mask_lon, mask_lat) - + if target_lons != mask_lons or target_lats != mask_lats: mask_var_regridded = interpolate.griddata((mask_lons.flatten(), mask_lats.flatten()), mask_var.flatten(), - (target_lons.flatten(), target_lats.flatten()), - method='nearest', + (target_lons.flatten(), + target_lats.flatten()), + method='nearest', fill_value=-9999.).reshape(target_lons.shape) else: mask_var_regridded = mask_var @@ -549,36 +557,38 @@ def regrid_spatial_mask(target_lon, target_lat, mask_lon, mask_lat, mask_var, mask_outside = ma.masked_equal(mask_var_regridded, -9999.).mask values_original = ma.array(mask_var) for shift in (-1, 1): - for axis in (0, 1): - q_shifted = np.roll(values_original, shift=shift, axis=axis) - idx = ~q_shifted.mask * values_original.mask - values_original.data[idx] = q_shifted[idx] + for axis in (0, 1): + q_shifted = np.roll(values_original, shift=shift, axis=axis) + idx = ~q_shifted.mask * values_original.mask + values_original.data[idx] = q_shifted[idx] # Make a masking map using nearest neighbour interpolation -use this to # determine locations with MDI and mask these qmdi = np.zeros_like(values_original) qmdi[values_original.mask == True] = 1. qmdi[values_original.mask == False] = 0. qmdi_r = map_coordinates(qmdi, [target_lats.flatten( - ), target_lons.flatten()], order=1).reshape(target_lons.shape) + ), target_lons.flatten()], order=1).reshape(target_lons.shape) mdimask = (qmdi_r != 0.0) - + for value in user_mask_values: mask_var_regridded = ma.masked_equal(mask_var_regridded, value) - + if extract: mask_out = np.invert(mask_var_regridded.mask | mdimask) return mask_out | mask_outside - else: - mask_out = mask_var_regridded.mask | mdimask + else: + mask_out = mask_var_regridded.mask | mdimask return mask_out | mask_outside + def propagate_spatial_mask_over_time(data_array, mask): if data_array.ndim == 3 and mask.ndim == 2: nt = data_array.shape[0] for it in np.arange(nt): mask_original = data_array[it, :].mask - data_array.mask[it,:] = mask | mask_original - return data_array + data_array.mask[it, :] = mask | mask_original + return data_array + def convert_lat_lon_2d_array(lon, lat): if lon.ndim == 1 and lat.ndim == 1: http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 0cda41f..72510cf 100644 --- a/setup.py +++ b/setup.py @@ -21,9 +21,9 @@ from setuptools import find_packages, setup # Package data # ------------ -_author = 'Apache Open Climate Workbench' -_author_email = 'd...@climate.apache.org' -_classifiers = [ +_author = 'Apache Open Climate Workbench' +_author_email = 'd...@climate.apache.org' +_classifiers = [ 'Environment :: Console', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', @@ -36,18 +36,19 @@ _classifiers = [ 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Python Modules', ] -_description = 'Apache Open Climate Workbench' -_download_url = 'http://pypi.python.org/pypi/ocw/' +_description = 'Apache Open Climate Workbench' +_download_url = 'http://pypi.python.org/pypi/ocw/' _requirements = [] -_keywords = ['climate analysis', 'workbench', 'rebinning', 'metrics', 'computation', 'visualization'] -_license = 'Apache License, Version 2.0' -_long_description = 'The Apache Open Climate Workbench provides tools for the evaluation and analysis of climate models.' -_name = 'ocw' -_namespaces = [] -_test_suite = 'ocw.tests' -_url = 'http://climate.apache.org/' -_version = '1.2.0' -_zip_safe = False +_keywords = ['climate analysis', 'workbench', 'rebinning', + 'metrics', 'computation', 'visualization'] +_license = 'Apache License, Version 2.0' +_long_description = 'The Apache Open Climate Workbench provides tools for the evaluation and analysis of climate models.' +_name = 'ocw' +_namespaces = [] +_test_suite = 'ocw.tests' +_url = 'http://climate.apache.org/' +_version = '1.2.0' +_zip_safe = False # Setup Metadata # -------------- http://git-wip-us.apache.org/repos/asf/climate/blob/731419f8/test_smoke.py ---------------------------------------------------------------------- diff --git a/test_smoke.py b/test_smoke.py index 863a943..b1599df 100644 --- a/test_smoke.py +++ b/test_smoke.py @@ -56,8 +56,8 @@ def check_dependencies(file): except VersionConflict as vc: fail(dep) print("\nRequired version and installed version differ for the " - "following package:\n" - "Required version: " + dep) + "following package:\n" + "Required version: " + dep) dep_name = str(vc).split(' ')[0][1:] # First element is '(' dep_version = str(vc).split(' ')[1] print("Installed version: " + dep_name + "==" + dep_version)
