This is an automated email from the ASF dual-hosted git repository. skperez pushed a commit to branch feature/SDAP-415 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit dbf61f23d45af63bf97d28a5b01ccf323edd4f39 Author: skorper <[email protected]> AuthorDate: Mon Mar 13 23:15:36 2023 -0700 fixed sat to sat matchup bug --- CHANGELOG.md | 1 + analysis/webservice/algorithms_spark/Matchup.py | 27 ++++++++++++++++--------- data-access/nexustiles/model/nexusmodel.py | 10 +++------ 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae01564..c108fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Made `platforms` param optional in `/cdmssubset`, and removed int requirement - Updated OpenAPI specification for `/cdmssubset` to accurately reflect `platforms` and `parameter` field options. - SDAP-436: Added special case for handling Cassandra SwathMulti tiles with uniform time arrays +- SDAP-415: Fixed bug where mask was incorrectly combined across all variables for multi-variable satellite to satellite matchup ### Security ## [1.0.0] - 2022-12-05 diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py index 5c5ec74..2bc91c3 100644 --- a/analysis/webservice/algorithms_spark/Matchup.py +++ b/analysis/webservice/algorithms_spark/Matchup.py @@ -169,10 +169,11 @@ class Matchup(NexusCalcSparkHandler): raise NexusProcessingException(reason="'secondary' argument is required", code=400) parameter_s = request.get_argument('parameter') - insitu_params = get_insitu_params(insitu_schema.get()) - if parameter_s and parameter_s not in insitu_params: - raise NexusProcessingException( - reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400) + if parameter_s: + insitu_params = get_insitu_params(insitu_schema.get()) + if parameter_s not in insitu_params: + raise NexusProcessingException( + reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400) try: start_time = request.get_start_datetime() @@ -801,31 +802,37 @@ def match_satellite_to_insitu(tile_ids, primary_b, secondary_b, parameter_b, tt_ polygon = Polygon( [(west, south), (east, south), (east, north), (west, north), (west, south)]) + # Find tile IDS from spatial/temporal bounds of partition matchup_tiles = tile_service.find_tiles_in_polygon( bounding_polygon=polygon, ds=secondary_b.value, start_time=matchup_min_time, end_time=matchup_max_time, - fetch_data=True, + fl='id', + fetch_data=False, sort=['tile_min_time_dt asc', 'tile_min_lon asc', 'tile_min_lat asc'], rows=5000 ) # Convert Tile IDS to tiles and convert to UTM lat/lon projection. matchup_points = [] + edge_results = [] for tile in matchup_tiles: + # Retrieve tile data and convert to lat/lon projection + tiles = tile_service.find_tile_by_id(tile.tile_id, fetch_data=True) + tile = tiles[0] + valid_indices = tile.get_indices() + primary_points = np.array([aeqd_proj( tile.longitudes[aslice[2]], tile.latitudes[aslice[1]] ) for aslice in valid_indices]) matchup_points.extend(primary_points) + edge_results.extend(tile_to_edge_points(tile)) - # Convert tiles to 'edge points' which match the format of in-situ edge points. - edge_results = [] - for matchup_tile in matchup_tiles: - edge_results.extend(tile_to_edge_points(matchup_tile)) - + if len(matchup_points) <= 0: + return [] matchup_points = np.array(matchup_points) print("%s Time to convert match points for partition %s to %s" % ( diff --git a/data-access/nexustiles/model/nexusmodel.py b/data-access/nexustiles/model/nexusmodel.py index f5c9df6..7db4d61 100644 --- a/data-access/nexustiles/model/nexusmodel.py +++ b/data-access/nexustiles/model/nexusmodel.py @@ -17,7 +17,7 @@ from collections import namedtuple import numpy as np from dataclasses import dataclass - +from functools import reduce NexusPoint = namedtuple('NexusPoint', 'latitude longitude depth time index data_vals') BBox = namedtuple('BBox', 'min_lat max_lat min_lon max_lon') @@ -156,12 +156,8 @@ class Tile(object): if include_nan: return list(np.ndindex(self.data.shape)) if self.is_multi: - # For each variable, combine masks. This is a logical or - # operation, because we want to ensure we don't lose any - # data. - combined_data_mask = np.logical_or(*self.data) - # Return the indices where the data is valid - return np.argwhere(combined_data_mask) + combined_data_inv_mask = reduce(np.logical_and, [data.mask for data in self.data]) + return np.argwhere(np.logical_not(combined_data_inv_mask)) else: return np.transpose(np.where(np.ma.getmaskarray(self.data) == False)).tolist()
