This is an automated email from the ASF dual-hosted git repository. rkk pushed a commit to branch SDAP-523 in repository https://gitbox.apache.org/repos/asf/sdap-ingester.git
commit be49d26d618ce46d8a0b9b04dad195f146679126 Author: rileykk <[email protected]> AuthorDate: Tue Jul 9 13:57:54 2024 -0700 Fixes for test case failures and excessive warnings --- CHANGELOG.md | 9 +++++++ .../processors/TileSummarizingProcessor.py | 9 +++++-- .../processors/test_ForceAscendingLatitude.py | 31 ++++++++++++---------- .../test_TileSummarizingProcessor.py | 6 ++--- .../test_GridMultiBandReadingProcessor.py | 17 +++++++----- .../test_GridReadingProcessor.py | 17 +++++++----- .../tests/slicers/test_SliceFileByStepSize.py | 10 ++++++- granule_ingester/tests/writers/test_SolrStore.py | 18 ++++++++----- 8 files changed, 77 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78a71db..6dad868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] - Unreleased +### Added +### Changed +### Deprecated +### Removed +### Fixed +- SDAP-523: Fixes for test case failures and excessive warnings in Ingester unit tests +### Security + ## [1.3.0] - 2024-06-10 ### Added - SDAP-472: Added support for defining Zarr collections in the collection config diff --git a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py index 041cac5..9eba094 100644 --- a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py +++ b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py @@ -62,8 +62,13 @@ class TileSummarizingProcessor(TileProcessor): tile_summary.bbox.lat_max = numpy.nanmax(latitudes).item() tile_summary.bbox.lon_min = numpy.nanmin(longitudes).item() tile_summary.bbox.lon_max = numpy.nanmax(longitudes).item() - tile_summary.stats.min = numpy.nanmin(data).item() - tile_summary.stats.max = numpy.nanmax(data).item() + + if all(numpy.isnan(data).flatten()): + tile_summary.stats.min = numpy.nan + tile_summary.stats.max = numpy.nan + else: + tile_summary.stats.min = numpy.nanmin(data).item() + tile_summary.stats.max = numpy.nanmax(data).item() tile_summary.stats.count = data.size - numpy.count_nonzero(numpy.isnan(data)) logger.debug(f'set summary fields') diff --git a/granule_ingester/tests/processors/test_ForceAscendingLatitude.py b/granule_ingester/tests/processors/test_ForceAscendingLatitude.py index 0d16d81..38a48ec 100644 --- a/granule_ingester/tests/processors/test_ForceAscendingLatitude.py +++ b/granule_ingester/tests/processors/test_ForceAscendingLatitude.py @@ -18,6 +18,7 @@ import unittest import xarray as xr import numpy as np from os import path +import warnings from granule_ingester.processors.reading_processors import GridMultiVariableReadingProcessor, GridReadingProcessor from nexusproto import DataTile_pb2 as nexusproto @@ -41,20 +42,22 @@ class TestForceAscendingLatitude(unittest.TestCase): 'lon': slice(0, 30) } - with xr.open_dataset(granule_path) as ds: - tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile) - flipped_tile = ForceAscendingLatitude().process(tile) - the_flipped_tile_type = flipped_tile.tile.WhichOneof("tile_type") - self.assertEqual(the_flipped_tile_type, 'grid_multi_variable_tile', f'wrong tile type') - the_flipped_tile_data = getattr(flipped_tile.tile, the_flipped_tile_type) - self.assertEqual([1, 30, 30, 2], the_flipped_tile_data.variable_data.shape) - flipped_latitudes = from_shaped_array(the_flipped_tile_data.latitude) - original_lat_data = ds['lat'].values - np.testing.assert_almost_equal(flipped_latitudes[0], original_lat_data[29], decimal=5, err_msg='wrong first vs last latitude', verbose=True) - np.testing.assert_almost_equal(flipped_latitudes[1], original_lat_data[28], decimal=5, err_msg='wrong latitude', verbose=True) - flipped_data = from_shaped_array(the_flipped_tile_data.variable_data) - original_b04_data = ds['B04'].values - np.testing.assert_almost_equal(original_b04_data[0][0][0], flipped_data[0][29][0][1], decimal=4, err_msg='wrong first vs last data', verbose=True) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', RuntimeWarning) + with xr.open_dataset(granule_path) as ds: + tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile) + flipped_tile = ForceAscendingLatitude().process(tile) + the_flipped_tile_type = flipped_tile.tile.WhichOneof("tile_type") + self.assertEqual(the_flipped_tile_type, 'grid_multi_variable_tile', f'wrong tile type') + the_flipped_tile_data = getattr(flipped_tile.tile, the_flipped_tile_type) + self.assertEqual([1, 30, 30, 2], the_flipped_tile_data.variable_data.shape) + flipped_latitudes = from_shaped_array(the_flipped_tile_data.latitude) + original_lat_data = ds['lat'].values + np.testing.assert_almost_equal(flipped_latitudes[0], original_lat_data[29], decimal=5, err_msg='wrong first vs last latitude', verbose=True) + np.testing.assert_almost_equal(flipped_latitudes[1], original_lat_data[28], decimal=5, err_msg='wrong latitude', verbose=True) + flipped_data = from_shaped_array(the_flipped_tile_data.variable_data) + original_b04_data = ds['B04'].values + np.testing.assert_almost_equal(original_b04_data[0][0][0], flipped_data[0][29][0][1], decimal=4, err_msg='wrong first vs last data', verbose=True) return def test_02_grid_single_band_data(self): diff --git a/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py b/granule_ingester/tests/processors/test_TileSummarizingProcessor.py similarity index 93% rename from granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py rename to granule_ingester/tests/processors/test_TileSummarizingProcessor.py index a0ac782..2d2641a 100644 --- a/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py +++ b/granule_ingester/tests/processors/test_TileSummarizingProcessor.py @@ -58,7 +58,7 @@ class TestTileSummarizingProcessor(unittest.TestCase): output_tile = reading_processor._generate_tile(ds, dims, input_tile) tile_summary_processor = TileSummarizingProcessor('test') new_tile = tile_summary_processor.process(tile=output_tile, dataset=ds) - self.assertEqual('"sea_surface_temperature"', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name') + self.assertEqual('["sea_surface_temperature"]', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name') def test_hls_single_var01(self): """ @@ -89,8 +89,8 @@ class TestTileSummarizingProcessor(unittest.TestCase): output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile) tile_summary_processor = TileSummarizingProcessor('test') new_tile = tile_summary_processor.process(tile=output_tile, dataset=ds) - self.assertEqual('null', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name') - self.assertEqual(None, json.loads(new_tile.summary.standard_name), f'unable to convert new_tile.summary.standard_name from JSON') + self.assertEqual('[null]', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name') + self.assertEqual([None], json.loads(new_tile.summary.standard_name), f'unable to convert new_tile.summary.standard_name from JSON') self.assertTrue(abs(new_tile.summary.stats.mean - 0.26137) < 0.001, f'mean value is not close expected: 0.26137. actual: {new_tile.summary.stats.mean}') def test_hls_multiple_var_01(self): diff --git a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py index 3c47718..291028a 100644 --- a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py +++ b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py @@ -14,6 +14,7 @@ # limitations under the License. import json import unittest +import warnings from os import path import numpy as np @@ -185,13 +186,15 @@ class TestCalendars(unittest.TestCase): tile = nexusproto.NexusTile() with xr.open_dataset(granule_path, decode_cf=True) as ds: - reading_processor = GridMultiVariableReadingProcessor( - data_vars, - lat_var, - lon_var, - time=time_var - ) - tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', RuntimeWarning) + reading_processor = GridMultiVariableReadingProcessor( + data_vars, + lat_var, + lon_var, + time=time_var + ) + tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile) assert tile.tile.grid_multi_variable_tile.time def test_julian_calendar_tile(self): diff --git a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py index 3123cc4..a0798fe 100644 --- a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py +++ b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py @@ -14,6 +14,7 @@ # limitations under the License. import unittest +import warnings from os import path import numpy as np @@ -422,13 +423,15 @@ class TestCalendars(unittest.TestCase): tile = nexusproto.NexusTile() with xr.open_dataset(granule_path, decode_cf=True) as ds: - reading_processor = GridReadingProcessor( - [data_var], - lat_var, - lon_var, - time=time_var - ) - tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', RuntimeWarning) + reading_processor = GridReadingProcessor( + [data_var], + lat_var, + lon_var, + time=time_var + ) + tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile) assert tile.tile.grid_tile.time def test_julian_calendar_tile(self): diff --git a/granule_ingester/tests/slicers/test_SliceFileByStepSize.py b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py index 5c091c1..81d032d 100644 --- a/granule_ingester/tests/slicers/test_SliceFileByStepSize.py +++ b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py @@ -48,7 +48,15 @@ class TestSliceFileByStepSize(unittest.TestCase): 'depth:2:4,latitude:180:360,longitude:540:720,nv:0:2,time:0:1' ] - self.assertEqual(expected_slices, slices) + # Neither the order of slices, nor the order of dimensions within the slices matter + + self.assertEqual(len(expected_slices), len(slices)) + + def slice_spec_to_tuples(s): + return tuple([(name, slc) for name, slc in sorted([dim.split(':', 1) for dim in s.split(',')])]) + + self.assertSetEqual(set([slice_spec_to_tuples(ex) for ex in expected_slices]), + set([slice_spec_to_tuples(s) for s in slices])) def test_generate_slices_indexed(self): netcdf_path = path.join(path.dirname(__file__), '../granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5') diff --git a/granule_ingester/tests/writers/test_SolrStore.py b/granule_ingester/tests/writers/test_SolrStore.py index 95a163a..2160c92 100644 --- a/granule_ingester/tests/writers/test_SolrStore.py +++ b/granule_ingester/tests/writers/test_SolrStore.py @@ -43,7 +43,8 @@ class TestSolrStore(unittest.TestCase): tile.summary.stats.max_time = 694310400 tile.summary.standard_name = json.dumps('sea_surface_temperature') - tile.tile.ecco_tile.depth = 10.5 + tile.tile.grid_tile.min_elevation = -10.5 + tile.tile.grid_tile.max_elevation = 0 metadata_store = SolrStore() solr_doc = metadata_store._build_solr_doc(tile) @@ -68,7 +69,8 @@ class TestSolrStore(unittest.TestCase): self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d']) self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d']) self.assertEqual(100, solr_doc['tile_count_i']) - self.assertAlmostEqual(10.5, solr_doc['tile_depth']) + self.assertAlmostEqual(-10.5, solr_doc['tile_min_elevation_d']) + self.assertAlmostEqual(0, solr_doc['tile_max_elevation_d']) def test_build_solr_doc_no_standard_name_02(self): tile = nexusproto.NexusTile() @@ -89,7 +91,8 @@ class TestSolrStore(unittest.TestCase): tile.summary.stats.min_time = 694224000 tile.summary.stats.max_time = 694310400 - tile.tile.ecco_tile.depth = 10.5 + tile.tile.grid_tile.min_elevation = -10.5 + tile.tile.grid_tile.max_elevation = 0 metadata_store = SolrStore() solr_doc = metadata_store._build_solr_doc(tile) @@ -113,7 +116,8 @@ class TestSolrStore(unittest.TestCase): self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d']) self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d']) self.assertEqual(100, solr_doc['tile_count_i']) - self.assertAlmostEqual(10.5, solr_doc['tile_depth']) + self.assertAlmostEqual(-10.5, solr_doc['tile_min_elevation_d']) + self.assertAlmostEqual(0, solr_doc['tile_max_elevation_d']) def test_build_solr_doc_no_standard_name(self): """ @@ -123,7 +127,8 @@ class TestSolrStore(unittest.TestCase): tile = nexusproto.NexusTile() tile.summary.tile_id = 'test_id' tile.summary.data_var_name = json.dumps(['test_variable', 'test_variable_02']) - tile.tile.ecco_tile.depth = 10.5 + tile.tile.grid_tile.min_elevation = -10.5 + tile.tile.grid_tile.max_elevation = 0 metadata_store = SolrStore() solr_doc = metadata_store._build_solr_doc(tile) @@ -142,7 +147,8 @@ class TestSolrStore(unittest.TestCase): tile.summary.tile_id = 'test_id' tile.summary.data_var_name = json.dumps(['test_variable', 'test_variable_02']) tile.summary.standard_name = json.dumps(['sea_surface_temperature', None]) - tile.tile.ecco_tile.depth = 10.5 + tile.tile.grid_tile.min_elevation = -10.5 + tile.tile.grid_tile.max_elevation = 0 metadata_store = SolrStore() solr_doc = metadata_store._build_solr_doc(tile)
