This is an automated email from the ASF dual-hosted git repository. rkk pushed a commit to branch tmp-stv in repository https://gitbox.apache.org/repos/asf/sdap-nexus.git
commit 050262894c43fb23a90e088b1710b9f73732041e Merge: 77400c8 28573fe Author: rileykk <[email protected]> AuthorDate: Fri Jul 5 13:14:22 2024 -0700 Merge remote-tracking branch 'origin/develop' into SDAP-492-tomograms # Conflicts: # analysis/webservice/webmodel/NexusRequestObject.py # data-access/nexustiles/backends/nexusproto/dao/SolrProxy.py # data-access/nexustiles/nexustiles.py .../webservice/algorithms_spark/ClimMapSpark.py | 2 +- .../webservice/algorithms_spark/CorrMapSpark.py | 2 +- .../DailyDifferenceAverageSpark.py | 14 +- .../webservice/algorithms_spark/HofMoellerSpark.py | 20 ++- .../algorithms_spark/MaximaMinimaSpark.py | 24 +++- .../webservice/algorithms_spark/TimeAvgMapSpark.py | 18 ++- .../webservice/algorithms_spark/TimeSeriesSpark.py | 3 +- .../webservice/algorithms_spark/VarianceSpark.py | 30 +++-- analysis/webservice/apidocs/openapi.yml | 144 +++++++++++++++++++++ .../request/handlers/NexusRequestHandler.py | 8 +- analysis/webservice/webmodel/NexusRequestObject.py | 54 ++++---- .../backends/nexusproto/dao/SolrProxy.py | 70 +++++----- data-access/nexustiles/nexustiles.py | 37 ++++++ 13 files changed, 332 insertions(+), 94 deletions(-) diff --cc analysis/webservice/webmodel/NexusRequestObject.py index 3cc7aeb,8b10886..0c159f1 --- a/analysis/webservice/webmodel/NexusRequestObject.py +++ b/analysis/webservice/webmodel/NexusRequestObject.py @@@ -115,36 -115,39 +115,39 @@@ class NexusRequestObject(StatsComputeOp def get_min_lon(self, default=Decimal(-180)): return self.get_decimal_arg("minLon", default) - def get_elevation_args(self) -> Tuple[Union[float, None], Union[float, None]]: + def get_elevation_args(self) -> Tuple[Optional[float], Optional[float]]: + ''' + Extract both single elevation args (depth, height, elevation) or + min/max elevation args (minDepth/maxDepth, minHeight/maxHeight, minElevation/maxElevation) + Return a tuple of the form (min, max) + ''' min_depth = self.get_float_arg('minDepth', None) max_depth = self.get_float_arg('maxDepth', None) + depth = self.get_float_arg('depth', None) - ++ min_height = self.get_float_arg('minHeight', None) max_height = self.get_float_arg('maxHeight', None) + height = self.get_float_arg('height', None) - ++ min_elevation = self.get_float_arg('minElevation', None) max_elevation = self.get_float_arg('maxElevation', None) - - ret_min = None - ret_max = None - using_depth = False - - if min_elevation is not None: - ret_min = min_elevation - elif min_height is not None: - ret_min = min_height - elif min_depth is not None: - ret_min = -1 * min_depth - using_depth = True - - if max_elevation is not None: - ret_max = max_elevation - elif max_height is not None: - ret_max = max_height - elif max_depth is not None: - ret_max = -1 * max_depth - using_depth = True - - if (ret_max is not None and ret_min is not None) and ret_max < ret_min: - if using_depth: + elevation = self.get_float_arg('elevation', None) + + # Handle single parameter cases + if depth is not None: + return -1 * depth, -1 * depth + elif height is not None: + return height, height + elif elevation is not None: + return elevation, elevation - ++ + # Handle min/max parameter cases + ret_min = min_elevation or min_height or (-1 * min_depth if min_depth is not None else None) + ret_max = max_elevation or max_height or (-1 * max_depth if max_depth is not None else None) + + # Validate max > min unless using depth args + if ret_max is not None and ret_min is not None and ret_max < ret_min: + if min_depth is not None or max_depth is not None: ret_max, ret_min = ret_min, ret_max else: raise ValueError(f'Request max elevation less than min elevation: {ret_max} < {ret_min}') diff --cc data-access/nexustiles/backends/nexusproto/dao/SolrProxy.py index b198140,e99df92..228b576 --- a/data-access/nexustiles/backends/nexusproto/dao/SolrProxy.py +++ b/data-access/nexustiles/backends/nexusproto/dao/SolrProxy.py @@@ -332,39 -332,12 +332,12 @@@ class SolrProxy(object) search_start_s, search_end_s ) additionalparams['fq'].append(time_clause) -- - min_elevation = kwargs['min_elevation'] if 'min_elevation' in kwargs else None - max_elevation = kwargs['max_elevation'] if 'max_elevation' in kwargs else None - - if min_elevation and max_elevation: - elevation_clause = "(" \ - "tile_min_elevation_d:[%s TO %s] " \ - "OR tile_max_elevation_d:[%s TO %s] " \ - "OR (tile_min_elevation_d:[* TO %s] AND tile_max_elevation_d:[%s TO *])" \ - ")" % ( - min_elevation, max_elevation, - min_elevation, max_elevation, - min_elevation, max_elevation - ) - additionalparams['fq'].append(elevation_clause) - elif min_elevation: - elevation_clause = "(" \ - "tile_min_elevation_d:[%s TO *] " \ - "OR tile_max_elevation_d:[%s TO *] " \ - ")" % ( - min_elevation, - min_elevation - ) - additionalparams['fq'].append(elevation_clause) - elif max_elevation: - elevation_clause = "(" \ - "tile_min_elevation_d:[* TO %s] " \ - "OR tile_max_elevation_d:[* TO %s] " \ - ")" % ( - max_elevation, - max_elevation - ) - additionalparams['fq'].append(elevation_clause) ++ + min_elevation = kwargs.get('min_elevation', None) + max_elevation = kwargs.get('max_elevation', None) - ++ + elevation_clause = self.get_elevation_clause(min_elevation, max_elevation) + additionalparams['fq'].append(elevation_clause) self._merge_kwargs(additionalparams, **kwargs) @@@ -399,7 -372,13 +372,13 @@@ search_start_s, search_end_s ) additionalparams['fq'].append(time_clause) - + + min_elevation = kwargs['min_elevation'] if 'min_elevation' in kwargs else None + max_elevation = kwargs['max_elevation'] if 'max_elevation' in kwargs else None - ++ + elevation_clause = self.get_elevation_clause(min_elevation, max_elevation) + additionalparams['fq'].append(elevation_clause) + self._merge_kwargs(additionalparams, **kwargs) return self.do_query_all( @@@ -433,7 -412,13 +412,13 @@@ search_start_s, search_end_s ) additionalparams['fq'].append(time_clause) - + + min_elevation = kwargs['min_elevation'] if 'min_elevation' in kwargs else None + max_elevation = kwargs['max_elevation'] if 'max_elevation' in kwargs else None - ++ + elevation_clause = self.get_elevation_clause(min_elevation, max_elevation) + additionalparams['fq'].append(elevation_clause) - ++ self._merge_kwargs(additionalparams, **kwargs) return self.do_query_all( @@@ -652,7 -637,26 +637,26 @@@ search_start_s, search_end_s ) return time_clause - + + def get_elevation_clause(self, min_elevation, max_elevation): + if min_elevation is not None and max_elevation is not None: + if min_elevation == max_elevation: + elevation_clause = f"(tile_min_elevation_d:[* TO {min_elevation}] AND tile_max_elevation_d:[{max_elevation} TO *])" + else: + elevation_clause = (f"(tile_min_elevation_d:[{min_elevation} TO {max_elevation}] OR " + f"tile_max_elevation_d:[{min_elevation} TO {max_elevation}] OR " + f"(tile_min_elevation_d:[* TO {min_elevation}] AND tile_max_elevation_d:[{max_elevation} TO *]))") + elif min_elevation is not None: + elevation_clause = (f"(tile_min_elevation_d:[{min_elevation} TO *] AND " + f"tile_max_elevation_d:[{min_elevation} TO *])") + elif max_elevation is not None: + elevation_clause = (f"(tile_min_elevation_d:[* TO {max_elevation}] AND " + f"tile_max_elevation_d:[* TO {max_elevation}])") + else: + elevation_clause = (f"((*:* -tile_min_elevation_d:[* TO *]) OR " + f"(tile_min_elevation_d:[0 TO 0] OR tile_max_elevation_d:[0 TO 0]))") + return elevation_clause - ++ def get_tile_count(self, ds, bounding_polygon=None, start_time=0, end_time=-1, metadata=None, **kwargs): """ Return number of tiles that match search criteria. diff --cc data-access/nexustiles/nexustiles.py index c491a4c,3ce3c28..323c279 --- a/data-access/nexustiles/nexustiles.py +++ b/data-access/nexustiles/nexustiles.py @@@ -718,44 -718,42 +718,81 @@@ class NexusTileService tile.data = ma.masked_where(data_mask, tile.data) tiles[:] = [tile for tile in tiles if not tile.data.mask.all()] + + return tiles + + def mask_tiles_to_elevation(self, min_e, max_e, tiles): + """ + Masks data in tiles to specified time range. + :param start_time: The start time to search for tiles + :param end_time: The end time to search for tiles + :param tiles: List of tiles + :return: A list tiles with data masked to specified time range + """ + if min_e is None: + min_e = -float('inf') + + if max_e is None: + max_e = float('inf') + + for tile in tiles: + if tile.elevation is None: + continue + + tile.elevation = ma.masked_outside(tile.elevation, min_e, max_e) + + data_mask = ma.getmaskarray(tile.elevation) + + # If this is multi-var, need to mask each variable separately. + if tile.is_multi: + # Combine space/time mask with existing mask on data + data_mask = reduce(np.logical_or, [tile.data[0].mask, data_mask]) + + num_vars = len(tile.data) + multi_data_mask = np.repeat(data_mask[np.newaxis, ...], num_vars, axis=0) + tile.data = ma.masked_where(multi_data_mask, tile.data) + else: + tile.data = ma.masked_where(data_mask, tile.data) + ++ tiles[:] = [tile for tile in tiles if not tile.data.mask.all()] + + return tiles + + def mask_tiles_to_elevation(self, min_e, max_e, tiles): + """ + Masks data in tiles to specified time range. + :param start_time: The start time to search for tiles + :param end_time: The end time to search for tiles + :param tiles: List of tiles + :return: A list tiles with data masked to specified time range + """ + if min_e is None: + min_e = -float('inf') + + if max_e is None: + max_e = float('inf') + + for tile in tiles: + if tile.elevation is None: + continue + + tile.elevation = ma.masked_outside(tile.elevation, min_e, max_e) + + data_mask = ma.getmaskarray(tile.elevation) + + # If this is multi-var, need to mask each variable separately. + if tile.is_multi: + # Combine space/time mask with existing mask on data + data_mask = reduce(np.logical_or, [tile.data[0].mask, data_mask]) + + num_vars = len(tile.data) + multi_data_mask = np.repeat(data_mask[np.newaxis, ...], num_vars, axis=0) + multi_data_mask = np.broadcast_arrays(multi_data_mask, tile.data)[0] + tile.data = ma.masked_where(multi_data_mask, tile.data) + else: + data_mask = np.broadcast_arrays(data_mask, tile.data)[0] + tile.data = ma.masked_where(data_mask, tile.data) + tiles[:] = [tile for tile in tiles if not tile.data.mask.all()] return tiles
