Repository: climate Updated Branches: refs/heads/master 088706d82 -> 386f7b924
http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_dataset.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_dataset.py b/ocw/tests/test_dataset.py index a51cddc..076c217 100644 --- a/ocw/tests/test_dataset.py +++ b/ocw/tests/test_dataset.py @@ -43,13 +43,13 @@ class TestDatasetAttributes(unittest.TestCase): origin=self.origin) def test_lats(self): - self.assertItemsEqual(self.test_dataset.lats, self.lat) + np.testing.assert_array_equal(self.test_dataset.lats, self.lat) def test_lons(self): - self.assertItemsEqual(self.test_dataset.lons, self.lon) + np.testing.assert_array_equal(self.test_dataset.lons, self.lon) def test_times(self): - self.assertItemsEqual(self.test_dataset.times, self.time) + np.testing.assert_array_equal(self.test_dataset.times, self.time) def test_values(self): self.assertEqual(self.test_dataset.values.all(), self.value.all()) http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_dataset_processor.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_dataset_processor.py b/ocw/tests/test_dataset_processor.py index 9538e91..9beee75 100644 --- a/ocw/tests/test_dataset_processor.py +++ b/ocw/tests/test_dataset_processor.py @@ -99,16 +99,16 @@ class TestTemporalRebinWithTimeIndex(unittest.TestCase): def test_time_dimension_multiple_of_orig_time_dimension(self): # ten_year_dataset.times.size is 120 - nt_avg = self.ten_year_dataset.times.size / 2 + nt_avg = self.ten_year_dataset.times.size // 2 # Temporal Rebin to exactly 2 (time) values dataset = dp.temporal_rebin_with_time_index( self.ten_year_dataset, nt_avg) start_time = self.ten_year_dataset.times[0] # First month of the middle year - middle_element = self.ten_year_dataset.times.size / 2 + middle_element = self.ten_year_dataset.times.size // 2 end_time = self.ten_year_dataset.times[middle_element] self.assertEqual(dataset.times.size, - self.ten_year_dataset.times.size / nt_avg) + self.ten_year_dataset.times.size // nt_avg) np.testing.assert_array_equal(dataset.times, [start_time, end_time]) def test_time_dimension_not_multiple_of_orig_time_dimension(self): @@ -119,7 +119,7 @@ class TestTemporalRebinWithTimeIndex(unittest.TestCase): self.ten_year_dataset, nt_avg) new_times = self.ten_year_dataset.times[::11][:-1] self.assertEqual(dataset.times.size, - self.ten_year_dataset.times.size / nt_avg) + self.ten_year_dataset.times.size // nt_avg) np.testing.assert_array_equal(dataset.times, new_times) def test_returned_dataset_attributes(self): http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_local.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_local.py b/ocw/tests/test_local.py index 645f486..bff2dd1 100644 --- a/ocw/tests/test_local.py +++ b/ocw/tests/test_local.py @@ -15,12 +15,17 @@ # specific language governing permissions and limitations # under the License. +# Needed Python 2/3 urllib compatability +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + import datetime import unittest import os -import urllib -import numpy import netCDF4 +import numpy as np import ocw.data_source.local as local @@ -51,42 +56,42 @@ class test_load_file(unittest.TestCase): def test_function_load_file_lats(self): """To test load_file function for latitudes""" - self.assertItemsEqual(local.load_file( + np.testing.assert_array_equal(local.load_file( self.file_path, "value").lats, self.latitudes) def test_function_load_file_lons(self): """To test load_file function for longitudes""" - self.assertItemsEqual(local.load_file( + np.testing.assert_array_equal(local.load_file( self.file_path, "value").lons, self.longitudes) def test_function_load_file_times(self): """To test load_file function for times""" newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) - self.assertItemsEqual(local.load_file( + np.testing.assert_array_equal(local.load_file( self.file_path, "value").times, newTimes) def test_function_load_file_alt_lats(self): """To test load_file function for lats with different variable names""" - self.assertItemsEqual(local.load_file( + np.testing.assert_array_equal(local.load_file( self.file_path, "value", lat_name="alt_lat").lats, self.alt_lats) def test_function_load_file_alt_lons(self): """To test load_file function for lons with different variable names""" - self.assertItemsEqual(local.load_file( + np.testing.assert_array_equal(local.load_file( self.file_path, "value", lon_name="alt_lon").lons, self.alt_lons) def test_function_load_file_alt_times(self): """To test load_file function for times with different variable names""" newTimes = datetime.datetime(2001, 4, 1), datetime.datetime( 2001, 5, 1), datetime.datetime(2001, 6, 1) - self.assertItemsEqual(local.load_file( + np.testing.assert_array_equal(local.load_file( self.file_path, "value", time_name="alt_time").times, newTimes) def test_function_load_file_values(self): """To test load_file function for values""" new_values = self.values[:, 0, :, :] - self.assertTrue(numpy.allclose(local.load_file( + self.assertTrue(np.allclose(local.load_file( self.file_path, "value").values, new_values)) def test_custom_dataset_name(self): @@ -125,7 +130,7 @@ class TestLoadMultipleFiles(unittest.TestCase): def test_function_load_multiple_files_lons(self): """To test load_multiple_file function for longitudes""" dataset = local.load_multiple_files(self.file_path, "value") - self.assertItemsEqual(dataset[0].lons, self.longitudes) + np.testing.assert_array_equal(dataset[0].lons, self.longitudes) def test_function_load_multiple_files_times(self): """To test load_multiple_files function for times""" @@ -133,14 +138,14 @@ class TestLoadMultipleFiles(unittest.TestCase): newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) - self.assertItemsEqual(dataset[0].times, newTimes) + np.testing.assert_array_equal(dataset[0].times, newTimes) def test_function_load_multiple_files_values(self): """To test load_multiple_files function for values""" new_values = self.values[:, 0, :, :] dataset = local.load_multiple_files( self.file_path, "value") - self.assertTrue(numpy.allclose(dataset[0].values, new_values)) + self.assertTrue(np.allclose(dataset[0].values, new_values)) def test_load_multiple_files_custom_dataset_name(self): """Test adding a custom name to a dataset""" @@ -192,40 +197,40 @@ class TestLoadDatasetFromMultipleNetcdfFiles(unittest.TestCase): def test_function_load_dataset_from_multiple_netcdf_files_lats(self): """To test load_multiple_files function for times""" - _, self.latitudes = numpy.meshgrid(self.longitudes, self.latitudes) - numpy.testing.assert_array_equal(self.dataset.lats, self.latitudes) + _, self.latitudes = np.meshgrid(self.longitudes, self.latitudes) + np.testing.assert_array_equal(self.dataset.lats, self.latitudes) def test_function_load_dataset_from_multiple_netcdf_files_lons(self): """To test load_multiple_files function for times""" - self.longitudes, _ = numpy.meshgrid(self.longitudes, self.latitudes) - numpy.testing.assert_array_equal(self.dataset.lons, self.longitudes) + self.longitudes, _ = np.meshgrid(self.longitudes, self.latitudes) + np.testing.assert_array_equal(self.dataset.lons, self.longitudes) def test_function_load_dataset_from_multiple_netcdf_files_times(self): """To test load_multiple_files function for times""" newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) - self.assertItemsEqual(self.dataset.times, newTimes) + np.testing.assert_array_equal(self.dataset.times, newTimes) def test_function_load_dataset_from_multiple_netcdf_files_alt_lats(self): """To test load_multiple_files function for non-default lats""" - _, self.alt_lats = numpy.meshgrid(self.alt_lons, self.alt_lats) - numpy.testing.assert_array_equal(self.alt_dataset.lats, self.alt_lats) + _, self.alt_lats = np.meshgrid(self.alt_lons, self.alt_lats) + np.testing.assert_array_equal(self.alt_dataset.lats, self.alt_lats) def test_function_load_dataset_from_multiple_netcdf_files_alt_lons(self): """To test load_multiple_files function for non-default lons""" - self.alt_lons, _ = numpy.meshgrid(self.alt_lons, self.alt_lats) - numpy.testing.assert_array_equal(self.alt_dataset.lons, self.alt_lons) + self.alt_lons, _ = np.meshgrid(self.alt_lons, self.alt_lats) + np.testing.assert_array_equal(self.alt_dataset.lons, self.alt_lons) def test_function_load_dataset_from_multiple_netcdf_files_alt_times(self): """To test load_multiple_files function for non-default times""" newTimes = datetime.datetime(2001, 4, 1), datetime.datetime( 2001, 5, 1), datetime.datetime(2001, 6, 1) - self.assertItemsEqual(self.alt_dataset.times, newTimes) + np.testing.assert_array_equal(self.alt_dataset.times, newTimes) def test_function_load_dataset_from_multiple_netcdf_files_values(self): """To test load_multiple_files function for values""" new_values = self.values[:, 0, :, :] - self.assertTrue(numpy.allclose(self.dataset.values, new_values)) + self.assertTrue(np.allclose(self.dataset.values, new_values)) class test_get_netcdf_variable_names(unittest.TestCase): @@ -233,7 +238,7 @@ class test_get_netcdf_variable_names(unittest.TestCase): test_model = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc" def setUp(self): - urllib.urlretrieve(self.file_path + self.test_model, self.test_model) + urlretrieve(self.file_path + self.test_model, self.test_model) self.invalid_netcdf_path = create_invalid_dimensions_netcdf_object() self.netcdf = netCDF4.Dataset(self.test_model, mode='r') @@ -295,14 +300,14 @@ def create_netcdf_object(): ) # To latitudes and longitudes for five values - latitudes_data = numpy.arange(5.) - longitudes_data = numpy.arange(150., 155.) + latitudes_data = np.arange(5.) + longitudes_data = np.arange(150., 155.) # Three months of data. - times_data = numpy.arange(3) + times_data = np.arange(3) # Two levels levels_data = [100, 200] # Create 150 values - values_data = numpy.array([i for i in range(150)]) + values_data = np.array([i for i in range(150)]) # Reshape values to 4D array (level, time, lats, lons) values_data = values_data.reshape(len(times_data), len( levels_data), len(latitudes_data), len(longitudes_data)) @@ -347,23 +352,23 @@ def create_invalid_dimensions_netcdf_object(): 'lon_dim') ) # To latitudes and longitudes for five values - latitudes = range(0, 5) - longitudes = range(200, 205) + flatitudes = list(range(0, 5)) + flongitudes = list(range(200, 205)) # Three months of data - times = range(3) + ftimes = list(range(3)) # Two levels - levels = [100, 200] + flevels = [100, 200] # Create 150 values - values = numpy.array([i for i in range(150)]) + fvalues = np.array([i for i in range(150)]) # Reshape values to 4D array (level, time, lats, lons) - values = values.reshape(len(levels), len( - times), len(latitudes), len(longitudes)) + fvalues = fvalues.reshape(len(flevels), len( + times), len(flatitudes), len(flongitudes)) # Ingest values to netCDF file - latitudes[:] = latitudes - longitudes[:] = longitudes - times[:] = times - levels[:] = levels - values[:] = values + latitudes[:] = flatitudes + longitudes[:] = flongitudes + times[:] = ftimes + levels[:] = flevels + values[:] = fvalues # Assign time info to time variable netCDF_file.variables['time'].units = 'months since 2001-01-01 00:00:00' netCDF_file.close() http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_plotter.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_plotter.py b/ocw/tests/test_plotter.py index 9020fd2..19ba66a 100644 --- a/ocw/tests/test_plotter.py +++ b/ocw/tests/test_plotter.py @@ -98,7 +98,7 @@ class TestFigshapeFunction(unittest.TestCase): gridshape = (2, 2) expected_width = 5.5 expected_height = 5.5 - width, height = plotter._fig_size(gridshape, aspect=(4 / 3)) + width, height = plotter._fig_size(gridshape, aspect=(4 // 3)) self.assertEqual(width, expected_width) self.assertEqual(height, expected_height) @@ -106,7 +106,7 @@ class TestFigshapeFunction(unittest.TestCase): gridshape = (567, 1223) expected_width = 11.0 expected_height = 5.5 - width, height = plotter._fig_size(gridshape, aspect=(16 / 9)) + width, height = plotter._fig_size(gridshape, aspect=(16 // 9)) self.assertEqual(width, expected_width) self.assertEqual(height, expected_height) http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/tests/test_rcmed.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_rcmed.py b/ocw/tests/test_rcmed.py index 35975a1..554776b 100644 --- a/ocw/tests/test_rcmed.py +++ b/ocw/tests/test_rcmed.py @@ -17,11 +17,9 @@ import unittest import datetime -import numpy +import numpy as np import pickle -import inspect import os -import test_rcmed # Import test_rcmed so we can use inspect to get the path import ocw.data_source.rcmed as rcmed @@ -47,33 +45,31 @@ class test_rcmed(unittest.TestCase, CustomAssertions): # of start_time and end_time self.start_time_for_url = "20020801T0000Z" self.end_time_for_url = "20021001T2359Z" - self.url = "http://rcmes.jpl.nasa.gov/query-api/query.php?"; - self.lats = numpy.arange(50.5, 70, 1) - self.lons = numpy.arange(1.5, 15, 1) + self.url = b"http://rcmes.jpl.nasa.gov/query-api/query.php?"; + self.lats = np.arange(50.5, 70, 1) + self.lons = np.arange(1.5, 15, 1) # In this parameter, two days of 10/20 and 10/21 have been missed. self.times_list = [datetime.datetime( 2002, 8, 31) + datetime.timedelta(days=x) for x in range(0, 62)] self.times_list.remove(datetime.datetime(2002, 10, 20)) self.times_list.remove(datetime.datetime(2002, 10, 21)) - self.times = numpy.array(self.times_list) + self.times = np.array(self.times_list) # Get the full path to test_rcmed - self.file_path = os.path.abspath(inspect.getfile(test_rcmed)) - # Remove the end of our path so we're left with the directory in which - # test_rcmed.py is stored. - self.file_path = self.file_path.split('test_rcmed')[0] + self.file_path = os.path.dirname(__file__) # Grab the parameter values file - params_file = open(os.path.join( - self.file_path, "parameters_values.p"), "rb") - self.values = pickle.load(params_file) + self.values = np.ma.array(np.load(os.path.join( + self.file_path, "parameter_values.npy"))) + # Grab the parameter metadata file meta_file = open(os.path.join( self.file_path, "parameters_metadata_output.p"), "rb") self.param_metadata_output = pickle.load(meta_file) def return_text(self, url): - if url == self.url + ("datasetId={0}&" + url = _force_bytes(url) + query = _force_bytes(("datasetId={0}&" "parameterId={1}&" "latMin={2}&" "latMax={3}&" @@ -87,17 +83,18 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.min_lon, self.max_lon, self.start_time_for_url, - self.end_time_for_url): + self.end_time_for_url)) + if url == self.url + query: return open(os.path.join(self.file_path, - "parameter_dataset_text.txt")) - elif url == self.url + "¶m_info=yes": + "parameter_dataset_text.txt"), 'rb') + elif url == self.url + b"¶m_info=yes": return open(os.path.join(self.file_path, - "parameters_metadata_text.txt")) + "parameters_metadata_text.txt"), 'rb') else: raise Exception("The URL is not acceptable.") def test_function_get_parameters_metadata(self): - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text self.assertEqual(rcmed.get_parameters_metadata(), self.param_metadata_output) @@ -105,7 +102,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.dataset_id = 6 self.parameter_id = 32 self.end_time_for_url = "20021031T0000Z" - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id, self.parameter_id, self.min_lat, @@ -117,7 +114,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.lats) def test_function_parameter_dataset_lats(self): - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id, self.parameter_id, self.min_lat, @@ -129,7 +126,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.lats) def test_function_parameter_dataset_lons(self): - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id, self.parameter_id, self.min_lat, @@ -141,7 +138,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.lons) def test_function_parameter_dataset_times(self): - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id, self.parameter_id, self.min_lat, @@ -153,7 +150,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.times) def test_function_parameter_dataset_values(self): - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text self.assert1DArraysEqual(rcmed.parameter_dataset(self.dataset_id, self.parameter_id, self.min_lat, @@ -166,7 +163,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.values.flatten()) def test_function_parameter_dataset_name(self): - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text ds = rcmed.parameter_dataset(self.dataset_id, self.parameter_id, self.min_lat, @@ -179,7 +176,7 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.assertEquals(ds.name, 'foo') def test_dataset_origin(self): - rcmed.urllib2.urlopen = self.return_text + rcmed.urlopen = self.return_text ds = rcmed.parameter_dataset(self.dataset_id, self.parameter_id, self.min_lat, @@ -194,5 +191,12 @@ class test_rcmed(unittest.TestCase, CustomAssertions): self.assertEquals(ds.origin['dataset_id'], self.dataset_id) self.assertEquals(ds.origin['parameter_id'], self.parameter_id) + +def _force_bytes(s, encoding='utf-8'): + if hasattr(s, 'encode'): + s = s.encode(encoding=encoding) + + return s + if __name__ == '__main__': unittest.main() http://git-wip-us.apache.org/repos/asf/climate/blob/528fe168/ocw/utils.py ---------------------------------------------------------------------- diff --git a/ocw/utils.py b/ocw/utils.py index 4876966..468ef58 100755 --- a/ocw/utils.py +++ b/ocw/utils.py @@ -364,7 +364,7 @@ def calc_time_series(dataset): ''' t_series = [] - for t in xrange(dataset.values.shape[0]): + for t in range(dataset.values.shape[0]): t_series.append(dataset.values[t, :, :].mean()) return t_series @@ -468,15 +468,25 @@ def shapefile_boundary(boundary_type, region_names): shapefile_dir = os.sep.join([os.path.dirname(__file__), 'shape']) map_read.readshapefile(os.path.join(shapefile_dir, boundary_type), boundary_type) + + # Note: The shapefile may contain countries or states with latin letters. + # Hence, the output of readshapefile can be a mix of ascii and unicode + # strings. Because python 2 and 3 treat them differently, we must + # explicitly check. if boundary_type == 'us_states': for region_name in region_names: + region_name = _force_unicode(region_name) for iregion, region_info in enumerate(map_read.us_states_info): - if region_info['st'] == region_name: + state = _force_unicode(region_info['st'], 'latin-1') + if state == region_name: regions.append(np.array(map_read.us_states[iregion])) elif boundary_type == 'countries': for region_name in region_names: + region_name = _force_unicode(region_name) for iregion, region_info in enumerate(map_read.countries_info): - if region_info['COUNTRY'].replace(" ", "").lower() == region_name.replace(" ", "").lower(): + country = _force_unicode(region_info['COUNTRY'], 'latin-1') + if (country.replace(" ", "").lower() == + region_name.replace(" ", "").lower()): regions.append(np.array(map_read.countries[iregion])) return regions @@ -595,3 +605,12 @@ def convert_lat_lon_2d_array(lon, lat): return np.meshgrid(lon, lat) if lon.ndim == 2 and lat.ndim == 2: return lon, lat + +def _force_unicode(s, encoding='utf-8'): + ''' + If the input is bytes, convert to unicode, otherwise return the input + ''' + if hasattr(s, 'decode'): + s = s.decode(encoding=encoding) + + return s
