This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/sdap-ingester.git
The following commit(s) were added to refs/heads/develop by this push:
new 58afa91 Fixes for test case failures and excessive warnings (#103)
58afa91 is described below
commit 58afa910550fdfebf97c161ebc6e0a42e991f8ab
Author: Riley Kuttruff <[email protected]>
AuthorDate: Tue Jul 9 14:14:02 2024 -0700
Fixes for test case failures and excessive warnings (#103)
Co-authored-by: rileykk <[email protected]>
---
CHANGELOG.md | 1 +
.../processors/TileSummarizingProcessor.py | 9 +++++--
.../processors/test_ForceAscendingLatitude.py | 31 ++++++++++++----------
.../test_TileSummarizingProcessor.py | 6 ++---
.../test_GridMultiBandReadingProcessor.py | 17 +++++++-----
.../test_GridReadingProcessor.py | 17 +++++++-----
.../tests/slicers/test_SliceFileByStepSize.py | 10 ++++++-
granule_ingester/tests/writers/test_SolrStore.py | 18 ++++++++-----
8 files changed, 69 insertions(+), 40 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5b9737b..a71c948 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic
Versioning](https://semver.org/spec/v2.0.0
### Deprecated
### Removed
### Fixed
+- SDAP-523: Fixes for test case failures and excessive warnings in Ingester
unit tests
### Security
## [1.3.0] - 2024-06-10
diff --git
a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
index 041cac5..9eba094 100644
--- a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
@@ -62,8 +62,13 @@ class TileSummarizingProcessor(TileProcessor):
tile_summary.bbox.lat_max = numpy.nanmax(latitudes).item()
tile_summary.bbox.lon_min = numpy.nanmin(longitudes).item()
tile_summary.bbox.lon_max = numpy.nanmax(longitudes).item()
- tile_summary.stats.min = numpy.nanmin(data).item()
- tile_summary.stats.max = numpy.nanmax(data).item()
+
+ if all(numpy.isnan(data).flatten()):
+ tile_summary.stats.min = numpy.nan
+ tile_summary.stats.max = numpy.nan
+ else:
+ tile_summary.stats.min = numpy.nanmin(data).item()
+ tile_summary.stats.max = numpy.nanmax(data).item()
tile_summary.stats.count = data.size -
numpy.count_nonzero(numpy.isnan(data))
logger.debug(f'set summary fields')
diff --git a/granule_ingester/tests/processors/test_ForceAscendingLatitude.py
b/granule_ingester/tests/processors/test_ForceAscendingLatitude.py
index 0d16d81..38a48ec 100644
--- a/granule_ingester/tests/processors/test_ForceAscendingLatitude.py
+++ b/granule_ingester/tests/processors/test_ForceAscendingLatitude.py
@@ -18,6 +18,7 @@ import unittest
import xarray as xr
import numpy as np
from os import path
+import warnings
from granule_ingester.processors.reading_processors import
GridMultiVariableReadingProcessor, GridReadingProcessor
from nexusproto import DataTile_pb2 as nexusproto
@@ -41,20 +42,22 @@ class TestForceAscendingLatitude(unittest.TestCase):
'lon': slice(0, 30)
}
- with xr.open_dataset(granule_path) as ds:
- tile = reading_processor._generate_tile(ds, dimensions_to_slices,
input_tile)
- flipped_tile = ForceAscendingLatitude().process(tile)
- the_flipped_tile_type = flipped_tile.tile.WhichOneof("tile_type")
- self.assertEqual(the_flipped_tile_type,
'grid_multi_variable_tile', f'wrong tile type')
- the_flipped_tile_data = getattr(flipped_tile.tile,
the_flipped_tile_type)
- self.assertEqual([1, 30, 30, 2],
the_flipped_tile_data.variable_data.shape)
- flipped_latitudes =
from_shaped_array(the_flipped_tile_data.latitude)
- original_lat_data = ds['lat'].values
- np.testing.assert_almost_equal(flipped_latitudes[0],
original_lat_data[29], decimal=5, err_msg='wrong first vs last latitude',
verbose=True)
- np.testing.assert_almost_equal(flipped_latitudes[1],
original_lat_data[28], decimal=5, err_msg='wrong latitude', verbose=True)
- flipped_data =
from_shaped_array(the_flipped_tile_data.variable_data)
- original_b04_data = ds['B04'].values
- np.testing.assert_almost_equal(original_b04_data[0][0][0],
flipped_data[0][29][0][1], decimal=4, err_msg='wrong first vs last data',
verbose=True)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', RuntimeWarning)
+ with xr.open_dataset(granule_path) as ds:
+ tile = reading_processor._generate_tile(ds,
dimensions_to_slices, input_tile)
+ flipped_tile = ForceAscendingLatitude().process(tile)
+ the_flipped_tile_type =
flipped_tile.tile.WhichOneof("tile_type")
+ self.assertEqual(the_flipped_tile_type,
'grid_multi_variable_tile', f'wrong tile type')
+ the_flipped_tile_data = getattr(flipped_tile.tile,
the_flipped_tile_type)
+ self.assertEqual([1, 30, 30, 2],
the_flipped_tile_data.variable_data.shape)
+ flipped_latitudes =
from_shaped_array(the_flipped_tile_data.latitude)
+ original_lat_data = ds['lat'].values
+ np.testing.assert_almost_equal(flipped_latitudes[0],
original_lat_data[29], decimal=5, err_msg='wrong first vs last latitude',
verbose=True)
+ np.testing.assert_almost_equal(flipped_latitudes[1],
original_lat_data[28], decimal=5, err_msg='wrong latitude', verbose=True)
+ flipped_data =
from_shaped_array(the_flipped_tile_data.variable_data)
+ original_b04_data = ds['B04'].values
+ np.testing.assert_almost_equal(original_b04_data[0][0][0],
flipped_data[0][29][0][1], decimal=4, err_msg='wrong first vs last data',
verbose=True)
return
def test_02_grid_single_band_data(self):
diff --git
a/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
b/granule_ingester/tests/processors/test_TileSummarizingProcessor.py
similarity index 93%
rename from
granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
rename to granule_ingester/tests/processors/test_TileSummarizingProcessor.py
index a0ac782..2d2641a 100644
--- a/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
+++ b/granule_ingester/tests/processors/test_TileSummarizingProcessor.py
@@ -58,7 +58,7 @@ class TestTileSummarizingProcessor(unittest.TestCase):
output_tile = reading_processor._generate_tile(ds, dims,
input_tile)
tile_summary_processor = TileSummarizingProcessor('test')
new_tile = tile_summary_processor.process(tile=output_tile,
dataset=ds)
- self.assertEqual('"sea_surface_temperature"',
new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name')
+ self.assertEqual('["sea_surface_temperature"]',
new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name')
def test_hls_single_var01(self):
"""
@@ -89,8 +89,8 @@ class TestTileSummarizingProcessor(unittest.TestCase):
output_tile = reading_processor._generate_tile(ds,
dimensions_to_slices, input_tile)
tile_summary_processor = TileSummarizingProcessor('test')
new_tile = tile_summary_processor.process(tile=output_tile,
dataset=ds)
- self.assertEqual('null', new_tile.summary.standard_name, f'wrong
new_tile.summary.standard_name')
- self.assertEqual(None, json.loads(new_tile.summary.standard_name),
f'unable to convert new_tile.summary.standard_name from JSON')
+ self.assertEqual('[null]', new_tile.summary.standard_name, f'wrong
new_tile.summary.standard_name')
+ self.assertEqual([None],
json.loads(new_tile.summary.standard_name), f'unable to convert
new_tile.summary.standard_name from JSON')
self.assertTrue(abs(new_tile.summary.stats.mean - 0.26137) <
0.001, f'mean value is not close expected: 0.26137. actual:
{new_tile.summary.stats.mean}')
def test_hls_multiple_var_01(self):
diff --git
a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
index 3c47718..291028a 100644
---
a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
+++
b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
@@ -14,6 +14,7 @@
# limitations under the License.
import json
import unittest
+import warnings
from os import path
import numpy as np
@@ -185,13 +186,15 @@ class TestCalendars(unittest.TestCase):
tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
- reading_processor = GridMultiVariableReadingProcessor(
- data_vars,
- lat_var,
- lon_var,
- time=time_var
- )
- tile = reading_processor._generate_tile(ds, dimensions_to_slices,
tile)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', RuntimeWarning)
+ reading_processor = GridMultiVariableReadingProcessor(
+ data_vars,
+ lat_var,
+ lon_var,
+ time=time_var
+ )
+ tile = reading_processor._generate_tile(ds,
dimensions_to_slices, tile)
assert tile.tile.grid_multi_variable_tile.time
def test_julian_calendar_tile(self):
diff --git
a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
index 3123cc4..a0798fe 100644
--- a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
@@ -14,6 +14,7 @@
# limitations under the License.
import unittest
+import warnings
from os import path
import numpy as np
@@ -422,13 +423,15 @@ class TestCalendars(unittest.TestCase):
tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
- reading_processor = GridReadingProcessor(
- [data_var],
- lat_var,
- lon_var,
- time=time_var
- )
- tile = reading_processor._generate_tile(ds, dimensions_to_slices,
tile)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', RuntimeWarning)
+ reading_processor = GridReadingProcessor(
+ [data_var],
+ lat_var,
+ lon_var,
+ time=time_var
+ )
+ tile = reading_processor._generate_tile(ds,
dimensions_to_slices, tile)
assert tile.tile.grid_tile.time
def test_julian_calendar_tile(self):
diff --git a/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
index 5c091c1..81d032d 100644
--- a/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
+++ b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
@@ -48,7 +48,15 @@ class TestSliceFileByStepSize(unittest.TestCase):
'depth:2:4,latitude:180:360,longitude:540:720,nv:0:2,time:0:1'
]
- self.assertEqual(expected_slices, slices)
+ # Neither the order of slices, nor the order of dimensions within
the slices matter
+
+ self.assertEqual(len(expected_slices), len(slices))
+
+ def slice_spec_to_tuples(s):
+ return tuple([(name, slc) for name, slc in
sorted([dim.split(':', 1) for dim in s.split(',')])])
+
+ self.assertSetEqual(set([slice_spec_to_tuples(ex) for ex in
expected_slices]),
+ set([slice_spec_to_tuples(s) for s in slices]))
def test_generate_slices_indexed(self):
netcdf_path = path.join(path.dirname(__file__),
'../granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5')
diff --git a/granule_ingester/tests/writers/test_SolrStore.py
b/granule_ingester/tests/writers/test_SolrStore.py
index 95a163a..2160c92 100644
--- a/granule_ingester/tests/writers/test_SolrStore.py
+++ b/granule_ingester/tests/writers/test_SolrStore.py
@@ -43,7 +43,8 @@ class TestSolrStore(unittest.TestCase):
tile.summary.stats.max_time = 694310400
tile.summary.standard_name = json.dumps('sea_surface_temperature')
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
@@ -68,7 +69,8 @@ class TestSolrStore(unittest.TestCase):
self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d'])
self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d'])
self.assertEqual(100, solr_doc['tile_count_i'])
- self.assertAlmostEqual(10.5, solr_doc['tile_depth'])
+ self.assertAlmostEqual(-10.5, solr_doc['tile_min_elevation_d'])
+ self.assertAlmostEqual(0, solr_doc['tile_max_elevation_d'])
def test_build_solr_doc_no_standard_name_02(self):
tile = nexusproto.NexusTile()
@@ -89,7 +91,8 @@ class TestSolrStore(unittest.TestCase):
tile.summary.stats.min_time = 694224000
tile.summary.stats.max_time = 694310400
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
@@ -113,7 +116,8 @@ class TestSolrStore(unittest.TestCase):
self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d'])
self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d'])
self.assertEqual(100, solr_doc['tile_count_i'])
- self.assertAlmostEqual(10.5, solr_doc['tile_depth'])
+ self.assertAlmostEqual(-10.5, solr_doc['tile_min_elevation_d'])
+ self.assertAlmostEqual(0, solr_doc['tile_max_elevation_d'])
def test_build_solr_doc_no_standard_name(self):
"""
@@ -123,7 +127,8 @@ class TestSolrStore(unittest.TestCase):
tile = nexusproto.NexusTile()
tile.summary.tile_id = 'test_id'
tile.summary.data_var_name = json.dumps(['test_variable',
'test_variable_02'])
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
@@ -142,7 +147,8 @@ class TestSolrStore(unittest.TestCase):
tile.summary.tile_id = 'test_id'
tile.summary.data_var_name = json.dumps(['test_variable',
'test_variable_02'])
tile.summary.standard_name = json.dumps(['sea_surface_temperature',
None])
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
