Repository: incubator-sdap-ningesterpy Updated Branches: refs/heads/master [created] e5513e07c
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/tilereadingprocessor_test.py ---------------------------------------------------------------------- diff --git a/tests/tilereadingprocessor_test.py b/tests/tilereadingprocessor_test.py new file mode 100644 index 0000000..2d69bbe --- /dev/null +++ b/tests/tilereadingprocessor_test.py @@ -0,0 +1,330 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import unittest +from os import path + +import numpy as np +from nexusproto.serialization import from_shaped_array + +import processors + + +class TestSummaryData(unittest.TestCase): + def setUp(self): + + self.module = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time') + + def test_summary_exists(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4') + + results = list( + self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('summary')) + + def test_section_spec_set(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4') + + results = list( + self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + # Tests for the first tile + self.assertEqual('time:0:1,lat:0:10,lon:0:10', results[0].summary.section_spec) + + # Tests for the second tile + self.assertEqual('time:0:1,lat:10:20,lon:0:10', results[1].summary.section_spec) + + def test_granule_set(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4') + + results = list( + self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + for nexus_tile in results: + self.assertEqual('empty_mur.nc4', nexus_tile.summary.granule) + + +class TestReadMurData(unittest.TestCase): + def setUp(self): + self.module = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time') + + def test_read_empty_mur(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4') + + results = list( + self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('grid_tile')) + + tile = nexus_tile.tile.grid_tile + self.assertEqual(10, len(from_shaped_array(tile.latitude))) + self.assertEqual(10, len(from_shaped_array(tile.longitude))) + + the_data = np.ma.masked_invalid(from_shaped_array(tile.variable_data)) + self.assertEqual((1, 10, 10), the_data.shape) + self.assertEqual(0, np.ma.count(the_data)) + self.assertTrue(tile.HasField('time')) + + def test_read_not_empty_mur(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_mur.nc4') + + results = list( + self.module.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data)) + self.assertEqual((1, 10, 10), tile1_data.shape) + self.assertEqual(100, np.ma.count(tile1_data)) + + tile2_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.grid_tile.variable_data)) + self.assertEqual((1, 10, 10), tile2_data.shape) + self.assertEqual(100, np.ma.count(tile2_data)) + + self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True), "Both tiles contain identical data") + + +class TestReadAscatbData(unittest.TestCase): + def setUp(self): + self.module = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time') + + # for data in read_swath_data(None, + # "NUMROWS:0:1,NUMCELLS:0:5;NUMROWS:1:2,NUMCELLS:0:5;file:///Users/greguska/data/ascat/ascat_20130314_004801_metopb_02520_eps_o_coa_2101_ovw.l2.nc"): + # import sys + # from struct import pack + # sys.stdout.write(pack("!L", len(data)) + data) + + # VARIABLE=wind_speed,LATITUDE=lat,LONGITUDE=lon,TIME=time,META=wind_dir,READER=SWATHTILE,TEMP_DIR=/tmp + def test_read_not_empty_ascatb(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4') + + swath_reader = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time') + + results = list( + swath_reader.process("NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('swath_tile')) + self.assertEqual(0, len(nexus_tile.tile.swath_tile.meta_data)) + + tile = nexus_tile.tile.swath_tile + self.assertEqual(82, from_shaped_array(tile.latitude).size) + self.assertEqual(82, from_shaped_array(tile.longitude).size) + + tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.variable_data)) + self.assertEqual((1, 82), tile1_data.shape) + self.assertEqual(82, np.ma.count(tile1_data)) + + tile2_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.swath_tile.variable_data)) + self.assertEqual((1, 82), tile2_data.shape) + self.assertEqual(82, np.ma.count(tile2_data)) + + self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True), "Both tiles contain identical data") + + def test_read_not_empty_ascatb_meta(self): + # with open('./ascat_longitude_more_than_180.bin', 'w') as f: + # results = list(self.module.read_swath_data(None, + # "NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file:///Users/greguska/Downloads/ascat_longitude_more_than_180.nc4")) + # f.write(results[0]) + + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4') + + swath_reader = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time', meta='wind_dir') + + results = list( + swath_reader.process("NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('swath_tile')) + self.assertLess(0, len(nexus_tile.tile.swath_tile.meta_data)) + + self.assertEqual(1, len(results[0].tile.swath_tile.meta_data)) + tile1_meta_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.meta_data[0].meta_data)) + self.assertEqual((1, 82), tile1_meta_data.shape) + self.assertEqual(82, np.ma.count(tile1_meta_data)) + + self.assertEqual(1, len(results[1].tile.swath_tile.meta_data)) + tile2_meta_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.swath_tile.meta_data[0].meta_data)) + self.assertEqual((1, 82), tile2_meta_data.shape) + self.assertEqual(82, np.ma.count(tile2_meta_data)) + + self.assertFalse(np.allclose(tile1_meta_data, tile2_meta_data, equal_nan=True), + "Both tiles' meta contain identical data") + + +class TestReadSmapData(unittest.TestCase): + def test_read_not_empty_smap(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_smap.h5') + + swath_reader = processors.SwathReadingProcessor('smap_sss', 'lat', 'lon', + time='row_time', + glblattr_day='REV_START_TIME', + glblattr_day_format='%Y-%jT%H:%M:%S.%f') + + results = list(swath_reader.process( + "phony_dim_0:0:76,phony_dim_1:0:1;phony_dim_0:0:76,phony_dim_1:1:2;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + # with open('./smap_nonempty_nexustile.bin', 'w') as f: + # f.write(results[0]) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('swath_tile')) + self.assertEqual(0, len(nexus_tile.tile.swath_tile.meta_data)) + + tile = nexus_tile.tile.swath_tile + self.assertEqual(76, from_shaped_array(tile.latitude).size) + self.assertEqual(76, from_shaped_array(tile.longitude).size) + + tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.variable_data)) + self.assertEqual((76, 1), tile1_data.shape) + self.assertEqual(43, np.ma.count(tile1_data)) + self.assertAlmostEqual(-50.056, + np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.latitude))), + places=3) + self.assertAlmostEqual(-47.949, + np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.latitude))), + places=3) + + tile2_data = np.ma.masked_invalid(from_shaped_array(results[1].tile.swath_tile.variable_data)) + self.assertEqual((76, 1), tile2_data.shape) + self.assertEqual(43, np.ma.count(tile2_data)) + + self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True), "Both tiles contain identical data") + + self.assertEqual(1427820162, np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.time))[0]) + + +class TestReadCcmpData(unittest.TestCase): + + def test_read_not_empty_ccmp(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ccmp.nc') + + ccmp_reader = processors.GridReadingProcessor('uwnd', 'latitude', 'longitude', time='time', meta='vwnd') + + results = list(ccmp_reader.process( + "time:0:1,longitude:0:87,latitude:0:38;time:1:2,longitude:0:87,latitude:0:38;time:2:3,longitude:0:87,latitude:0:38;time:3:4,longitude:0:87,latitude:0:38;file://%s" % test_file)) + + self.assertEqual(4, len(results)) + + # with open('./ccmp_nonempty_nexustile.bin', 'w') as f: + # f.write(results[0]) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('grid_tile')) + self.assertEqual(1, len(nexus_tile.tile.grid_tile.meta_data)) + + tile = nexus_tile.tile.grid_tile + self.assertEqual(38, from_shaped_array(tile.latitude).size) + self.assertEqual(87, from_shaped_array(tile.longitude).size) + self.assertEqual((1, 38, 87), from_shaped_array(tile.variable_data).shape) + + tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data)) + self.assertEqual(3306, np.ma.count(tile1_data)) + self.assertAlmostEqual(-78.375, + np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))), + places=3) + self.assertAlmostEqual(-69.125, + np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))), + places=3) + + self.assertEqual(1451606400, results[0].tile.grid_tile.time) + + +class TestReadAvhrrData(unittest.TestCase): + def test_read_not_empty_avhrr(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_avhrr.nc4') + + avhrr_reader = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time') + + results = list(avhrr_reader.process("time:0:1,lat:0:10,lon:0:10;file://%s" % test_file)) + + self.assertEqual(1, len(results)) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('grid_tile')) + + tile = nexus_tile.tile.grid_tile + self.assertEqual(10, from_shaped_array(tile.latitude).size) + self.assertEqual(10, from_shaped_array(tile.longitude).size) + self.assertEqual((1, 10, 10), from_shaped_array(tile.variable_data).shape) + + tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data)) + self.assertEqual(100, np.ma.count(tile1_data)) + self.assertAlmostEqual(-39.875, + np.ma.min(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))), + places=3) + self.assertAlmostEqual(-37.625, + np.ma.max(np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.latitude))), + places=3) + + self.assertEqual(1462060800, results[0].tile.grid_tile.time) + self.assertAlmostEqual(289.71, + np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))[ + 0, 0, 0], + places=3) + + +class TestReadWSWMData(unittest.TestCase): + + def test_read_not_empty_wswm(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_wswm.nc') + + wswm_reader = processors.TimeSeriesReadingProcessor('Qout', 'lat', 'lon', 'time') + + results = list(wswm_reader.process("time:0:1,rivid:0:500;time:0:1,rivid:500:1000;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + for nexus_tile in results: + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('time_series_tile')) + + tile = nexus_tile.tile.time_series_tile + self.assertEqual(500, from_shaped_array(tile.latitude).size) + self.assertEqual(500, from_shaped_array(tile.longitude).size) + self.assertEqual((1, 500), from_shaped_array(tile.variable_data).shape) + + tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.variable_data)) + self.assertEqual(500, np.ma.count(tile1_data)) + self.assertAlmostEqual(41.390, + np.ma.min( + np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.latitude))), + places=3) + self.assertAlmostEqual(42.071, + np.ma.max( + np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.latitude))), + places=3) + + self.assertEqual(852098400, results[0].tile.time_series_tile.time) + self.assertAlmostEqual(0.009, + np.ma.masked_invalid(from_shaped_array(results[0].tile.time_series_tile.variable_data))[ + 0, 0], + places=3) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/tilesumarizingprocessor_test.py ---------------------------------------------------------------------- diff --git a/tests/tilesumarizingprocessor_test.py b/tests/tilesumarizingprocessor_test.py new file mode 100644 index 0000000..b9735d8 --- /dev/null +++ b/tests/tilesumarizingprocessor_test.py @@ -0,0 +1,80 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import unittest +from os import path + +import processors + + +class TestSummarizeTile(unittest.TestCase): + def test_summarize_swath(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'smap_nonempty_nexustile.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + summarizer = processors.TileSummarizingProcessor() + + results = list(summarizer.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile = results[0] + + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('swath_tile')) + self.assertTrue(nexus_tile.HasField('summary')) + + # Check summary + tile_summary = nexus_tile.summary + self.assertAlmostEqual(-50.056, tile_summary.bbox.lat_min, places=3) + self.assertAlmostEqual(-47.949, tile_summary.bbox.lat_max, places=3) + self.assertAlmostEqual(22.376, tile_summary.bbox.lon_min, places=3) + self.assertAlmostEqual(36.013, tile_summary.bbox.lon_max, places=3) + + self.assertAlmostEqual(33.067, tile_summary.stats.min, places=3) + self.assertEqual(40, tile_summary.stats.max) + self.assertAlmostEqual(36.6348, tile_summary.stats.mean, places=3) + self.assertEqual(43, tile_summary.stats.count) + + self.assertEqual(1427820162, tile_summary.stats.min_time) + self.assertEqual(1427820162, tile_summary.stats.max_time) + + def test_summarize_grid(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'avhrr_nonempty_nexustile.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + summarizer = processors.TileSummarizingProcessor() + + results = list(summarizer.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile = results[0] + + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('grid_tile')) + self.assertTrue(nexus_tile.HasField('summary')) + + # Check summary + tile_summary = nexus_tile.summary + self.assertAlmostEqual(-39.875, tile_summary.bbox.lat_min, places=3) + self.assertAlmostEqual(-37.625, tile_summary.bbox.lat_max, places=3) + self.assertAlmostEqual(-129.875, tile_summary.bbox.lon_min, places=3) + self.assertAlmostEqual(-127.625, tile_summary.bbox.lon_max, places=3) + + self.assertAlmostEqual(288.5099, tile_summary.stats.min, places=3) + self.assertAlmostEqual(290.4, tile_summary.stats.max, places=3) + self.assertAlmostEqual(289.4443, tile_summary.stats.mean, places=3) + self.assertEqual(100, tile_summary.stats.count) + + self.assertEqual(1462838400, tile_summary.stats.min_time) + self.assertEqual(1462838400, tile_summary.stats.max_time) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/winddirspeedtouv_test.py ---------------------------------------------------------------------- diff --git a/tests/winddirspeedtouv_test.py b/tests/winddirspeedtouv_test.py new file mode 100644 index 0000000..bb87d81 --- /dev/null +++ b/tests/winddirspeedtouv_test.py @@ -0,0 +1,89 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import unittest +from os import path + +import numpy as np +from nexusproto.serialization import from_shaped_array + +import processors + + +class TestAscatbUData(unittest.TestCase): + + def test_u_conversion(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascatb_nonempty_nexustile.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + converter = processors.WindDirSpeedToUV('U') + + results = list(converter.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile = results[0] + + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('swath_tile')) + + # Check data + tile_data = np.ma.masked_invalid(from_shaped_array(nexus_tile.tile.swath_tile.variable_data)) + self.assertEqual(82, np.ma.count(tile_data)) + + # Check meta data + meta_list = nexus_tile.tile.swath_tile.meta_data + self.assertEqual(3, len(meta_list)) + wind_dir = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_dir') + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape) + self.assertIsNotNone(wind_dir) + wind_speed = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_speed') + self.assertIsNotNone(wind_speed) + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_speed.meta_data)).shape) + wind_v = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_v') + self.assertIsNotNone(wind_v) + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_v.meta_data)).shape) + + +class TestAscatbVData(unittest.TestCase): + + def test_u_conversion(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascatb_nonempty_nexustile.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + converter = processors.WindDirSpeedToUV('V') + + results = list(converter.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile = results[0] + + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('swath_tile')) + + # Check data + tile_data = np.ma.masked_invalid(from_shaped_array(nexus_tile.tile.swath_tile.variable_data)) + self.assertEqual(82, np.ma.count(tile_data)) + + # Check meta data + meta_list = nexus_tile.tile.swath_tile.meta_data + self.assertEqual(3, len(meta_list)) + wind_dir = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_dir') + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape) + self.assertIsNotNone(wind_dir) + wind_speed = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_speed') + self.assertIsNotNone(wind_speed) + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_speed.meta_data)).shape) + wind_u = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_u') + self.assertIsNotNone(wind_u) + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_u.meta_data)).shape) + + +if __name__ == '__main__': + unittest.main()
