This is an automated email from the ASF dual-hosted git repository.
jjacob pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new d9cf0fa SDAP-351: Fix reading of data with cftime.datetime.Gregorian
type (#46)
d9cf0fa is described below
commit d9cf0faaeec4ec7b24410a93605191cb20ad7e3b
Author: Stepheny Perez <[email protected]>
AuthorDate: Wed Nov 10 13:56:23 2021 -0800
SDAP-351: Fix reading of data with cftime.datetime.Gregorian type (#46)
* Generalized datetime fix so all calendars are supported
* Added unit tests for 4 calendar types (standard, missing attr, julian,
gregorian
---
.../GridMultiVariableReadingProcessor.py | 2 +-
.../reading_processors/GridReadingProcessor.py | 2 +-
...JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc | Bin 0 -> 1854879 bytes
.../20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc | Bin 0 -> 100835190 bytes
...SS_L4_multimission_global_7d_v1.0_2021-03-12.nc | Bin 0 -> 8307796 bytes
.../test_GridMultiBandReadingProcessor.py | 66 +++++++++++++++++----
.../test_GridReadingProcessor.py | 65 +++++++++++++++++---
7 files changed, 114 insertions(+), 21 deletions(-)
diff --git
a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
index dc28c19..c36b8d2 100644
---
a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
+++
b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
@@ -74,7 +74,7 @@ class GridMultiVariableReadingProcessor(TileReadingProcessor):
"Time slices must have length 1, but '{dim}' has length
{dim_len}.".format(dim=self.time,
dim_len=time_slice_len))
- if isinstance(ds[self.time][time_slice.start].item(),
cftime.DatetimeJulian):
+ if isinstance(ds[self.time][time_slice.start].item(),
cftime.datetime):
ds[self.time] = ds.indexes[self.time].to_datetimeindex()
new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9)
diff --git
a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
index 725ba35..97f282c 100644
---
a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
+++
b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
@@ -47,7 +47,7 @@ class GridReadingProcessor(TileReadingProcessor):
raise RuntimeError(
"Time slices must have length 1, but '{dim}' has length
{dim_len}.".format(dim=self.time,
dim_len=time_slice_len))
- if isinstance(ds[self.time][time_slice.start].item(),
cftime.DatetimeJulian):
+ if isinstance(ds[self.time][time_slice.start].item(),
cftime.datetime):
ds[self.time] = ds.indexes[self.time].to_datetimeindex()
new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9)
diff --git
a/granule_ingester/tests/granules/20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc
b/granule_ingester/tests/granules/20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc
new file mode 100644
index 0000000..31ca293
Binary files /dev/null and
b/granule_ingester/tests/granules/20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc
differ
diff --git
a/granule_ingester/tests/granules/20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc
b/granule_ingester/tests/granules/20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc
new file mode 100644
index 0000000..6fd6cd3
Binary files /dev/null and
b/granule_ingester/tests/granules/20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc
differ
diff --git
a/granule_ingester/tests/granules/OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc
b/granule_ingester/tests/granules/OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc
new file mode 100644
index 0000000..283bfe4
Binary files /dev/null and
b/granule_ingester/tests/granules/OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc
differ
diff --git
a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
index 607dd8b..3c47718 100644
---
a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
+++
b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
@@ -165,27 +165,71 @@ class TestReadHLSData(unittest.TestCase):
self.assertRaises(RuntimeError, GridMultiVariableReadingProcessor, [],
'lat', 'lon', time='time')
return
-class TestReadGPMData(unittest.TestCase):
- def test_generate_tile(self):
- granule_path = path.join(path.dirname(__file__),
'../granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4')
+
+class TestCalendars(unittest.TestCase):
+ """
+ Test that various calendars can be ingested into SDAP without error.
+ """
+ def assert_time_data(self, granule_name, time_var, lat_var, lon_var,
data_vars):
+ granule_path = path.join(
+ path.dirname(__file__),
+ '../granules/',
+ granule_name
+ )
dimensions_to_slices = {
- 'time': slice(0, 1),
- 'lat': slice(0, 30),
- 'lon': slice(0, 30)
+ time_var: slice(0, 1),
+ lat_var: slice(0, 30),
+ lon_var: slice(0, 30)
}
- tile = nexusproto.NexusTile()
+ tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
reading_processor = GridMultiVariableReadingProcessor(
- ['HQprecipitation', 'HQprecipitation_cnt'],
- 'lat',
- 'lon',
- time='time'
+ data_vars,
+ lat_var,
+ lon_var,
+ time=time_var
)
tile = reading_processor._generate_tile(ds, dimensions_to_slices,
tile)
assert tile.tile.grid_multi_variable_tile.time
+ def test_julian_calendar_tile(self):
+ self.assert_time_data(
+
granule_name='3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_vars=['HQprecipitation', 'HQprecipitation_cnt']
+ )
+
+ def test_gregorian_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_vars=['CHL', 'CHL_error']
+ )
+
+ def test_standard_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc',
+ time_var='time',
+ lat_var='latitude',
+ lon_var='longitude',
+ data_vars=['sss', 'sss_uncertainty']
+ )
+
+ def test_missing_calendar_tile(self):
+ self.assert_time_data(
+
granule_name='20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_vars=['analysed_sst', 'analysis_error']
+ )
+
if __name__ == '__main__':
unittest.main()
diff --git
a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
index 6cfa835..ef78394 100644
--- a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
@@ -347,22 +347,71 @@ class TestReadHLSData(unittest.TestCase):
self.assertRaises(RuntimeError, GridReadingProcessor, [], 'lat',
'lon', time='time')
return
-class TestReadGPMData(unittest.TestCase):
- def test_generate_tile(self):
- granule_path = path.join(path.dirname(__file__),
'../granules/3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4')
+class TestCalendars(unittest.TestCase):
+ """
+ Test that various calendars can be ingested into SDAP without error.
+ """
+
+ def assert_time_data(self, granule_name, time_var, lat_var, lon_var,
data_var):
+ granule_path = path.join(
+ path.dirname(__file__),
+ '../granules/',
+ granule_name
+ )
dimensions_to_slices = {
- 'time': slice(0, 1),
- 'lat': slice(0, 30),
- 'lon': slice(0, 30)
+ time_var: slice(0, 1),
+ lat_var: slice(0, 30),
+ lon_var: slice(0, 30)
}
- tile = nexusproto.NexusTile()
+ tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
- reading_processor = GridReadingProcessor(['HQprecipitation'],
'lat', 'lon', time='time')
+ reading_processor = GridReadingProcessor(
+ [data_var],
+ lat_var,
+ lon_var,
+ time=time_var
+ )
tile = reading_processor._generate_tile(ds, dimensions_to_slices,
tile)
assert tile.tile.grid_tile.time
+ def test_julian_calendar_tile(self):
+ self.assert_time_data(
+
granule_name='3B-DAY-E.MS.MRG.3IMERG.20070101-S000000-E235959.V06.nc4',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_var='HQprecipitation'
+ )
+
+ def test_gregorian_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='20190630_d-ACRI-L4-CHL-MULTI_4KM-GLO-REP.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_var='CHL'
+ )
+
+ def test_standard_calendar_tile(self):
+ self.assert_time_data(
+ granule_name='OISSS_L4_multimission_global_7d_v1.0_2021-03-12.nc',
+ time_var='time',
+ lat_var='latitude',
+ lon_var='longitude',
+ data_var='sss'
+ )
+
+ def test_missing_calendar_tile(self):
+ self.assert_time_data(
+
granule_name='20181231090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc',
+ time_var='time',
+ lat_var='lat',
+ lon_var='lon',
+ data_var='analysed_sst'
+ )
+
if __name__ == '__main__':
unittest.main()
