This is an automated email from the ASF dual-hosted git repository. rkk pushed a commit to branch SDAP-502 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f7d8c670fdb4134fe2d427672bf8e5b509e75f22 Author: rileykk <[email protected]> AuthorDate: Wed Jan 17 08:27:18 2024 -0800 SDAP-502: Gridded tile gen squeeze bug patch --- CHANGELOG.md | 1 + .../GridMultiVariableReadingProcessor.py | 13 +++++++++++-- .../reading_processors/GridReadingProcessor.py | 21 +++++++++++++++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929e6c4..01e5136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - SDAP-501: Updated dependencies to remove `chardet` ### Fixed - SDAP-488: Workaround to build issue on Apple Silicon (M1/M2). GI image build installs nexusproto through PyPI instead of building from source. A build arg `BUILD_NEXUSPROTO` was defined to allow building from source if desired/ +- SDAP-502: Fix for rare bug where gridded tiles generated from inputs where there is a dimension length where `dimensionLenght mod tileSliceLenght == 1` would cause tile generation to fail. This is because `np.squeeze` is used on the coordinate arrays, which, if the generated tile has only a single lat or lon, would squeeze the corresponding coordinate into a dimensionless array, which would raise an error down the line when `len` was called with it. Added a check for this case that bot [...] ### Security ## [1.1.0] - 2023-04-26 diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py index 035bb6b..bab04f6 100644 --- a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py +++ b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py @@ -55,8 +55,17 @@ class GridMultiVariableReadingProcessor(TileReadingProcessor): lat_subset = ds[self.latitude][type(self)._slices_for_variable(ds[self.latitude], dimensions_to_slices)] lon_subset = ds[self.longitude][type(self)._slices_for_variable(ds[self.longitude], dimensions_to_slices)] - lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN) - lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN) + + lat_subset = np.squeeze(lat_subset) + if lat_subset.shape == (): + lat_subset = np.expand_dims(lat_subset, 0) + + lon_subset = np.squeeze(lon_subset) + if lon_subset.shape == (): + lon_subset = np.expand_dims(lon_subset, 0) + + lat_subset = np.ma.filled(lat_subset, np.NaN) + lon_subset = np.ma.filled(lon_subset, np.NaN) if not isinstance(self.variable, list): raise ValueError(f'self.variable `{self.variable}` needs to be a list. use GridReadingProcessor for single band Grid files.') diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py index 73969e6..95bb7c6 100644 --- a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py +++ b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py @@ -36,15 +36,32 @@ class GridReadingProcessor(TileReadingProcessor): data_variable = self.variable[0] if isinstance(self.variable, list) else self.variable new_tile = nexusproto.GridTile() + expand_axes = [] + lat_subset = ds[self.latitude][type(self)._slices_for_variable(ds[self.latitude], dimensions_to_slices)] lon_subset = ds[self.longitude][type(self)._slices_for_variable(ds[self.longitude], dimensions_to_slices)] - lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN) - lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN) + + lat_subset = np.squeeze(lat_subset) + if lat_subset.shape == (): + lat_subset = np.expand_dims(lat_subset, 0) + expand_axes.append(0) + + lon_subset = np.squeeze(lon_subset) + if lon_subset.shape == (): + lon_subset = np.expand_dims(lon_subset, 0) + expand_axes.append(1) + + lat_subset = np.ma.filled(lat_subset, np.NaN) + lon_subset = np.ma.filled(lon_subset, np.NaN) data_subset = ds[data_variable][type(self)._slices_for_variable(ds[data_variable], dimensions_to_slices)].data data_subset = np.array(np.squeeze(data_subset)) + if len(expand_axes) > 0: + data_subset = np.expand_dims(data_subset, tuple(expand_axes)) + + if self.depth: depth_dim, depth_slice = list(type(self)._slices_for_variable(ds[self.depth], dimensions_to_slices).items())[0]
