This is an automated email from the ASF dual-hosted git repository. skperez pushed a commit to branch proxy-bugfix-apidocs in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 2bab8cb23b23487848e24c310834e8e96be25d98 Author: skorper <[email protected]> AuthorDate: Mon Aug 22 08:22:58 2022 -0700 Updated matchup output fields. Validate matchup 'parameter' param against insitu schema --- CHANGELOG.md | 4 ++- analysis/webservice/algorithms_spark/Matchup.py | 39 ++++++++++++++----------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa47d27..3d97eef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,13 +14,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - SDAP-371: Renamed `/domssubset` endpoint to `/cdmssubset` - SDAP-390: Updated NetCDF reader tool for data matchup and added user functionality. - SDAP-396: Added saildrone insitu api to matchup +- Matchup validates insitu parameter using insitu API schema endpoint ### Changed - -SDAP-390: Changed `/doms` to `/cdms` and `doms_reader.py` to `cdms_reader.py` - domslist endpoint points to AWS insitu instead of doms insitu +- Matchup returns numSecondary and numPrimary counts rather than insitu/gridded ### Deprecated ### Removed - removed dropdown from matchup doms endpoint secondary param +- Matchup no longer returns insituMatched and griddedMatched fields ### Fixed - Fix failing test_matchup unit test - Fixed bug in OpenAPI spec where both matchup endpoints shared the same id diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py index b85c7b4..5dafb31 100644 --- a/analysis/webservice/algorithms_spark/Matchup.py +++ b/analysis/webservice/algorithms_spark/Matchup.py @@ -15,7 +15,6 @@ from typing import Optional -import json import logging import threading from shapely.geometry import Polygon @@ -30,9 +29,7 @@ import requests from pytz import timezone, UTC from scipy import spatial from shapely import wkt -from shapely.geometry import Point from shapely.geometry import box -from shapely.geos import WKTReadingError from webservice.NexusHandler import nexus_handler from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSparkHandler @@ -46,6 +43,7 @@ from webservice.webmodel import NexusProcessingException EPOCH = timezone('UTC').localize(datetime(1970, 1, 1)) ISO_8601 = '%Y-%m-%dT%H:%M:%S%z' +insitu_schema = query_insitu_schema() def iso_time_to_epoch(str_time): @@ -73,7 +71,7 @@ class Matchup(NexusCalcSparkHandler): "parameter": { "name": "Match-Up Parameter", "type": "string", - "description": "The parameter of interest used for the match up. One of 'sst', 'sss', 'wind'. Optional" + "description": "The parameter of interest used for the match up. Only used for satellite to insitu matchups. Optional" }, "startTime": { "name": "Start Time", @@ -157,9 +155,10 @@ class Matchup(NexusCalcSparkHandler): raise NexusProcessingException(reason="'secondary' argument is required", code=400) parameter_s = request.get_argument('parameter') - if parameter_s and parameter_s not in ['sst', 'sss', 'wind']: + insitu_params = get_insitu_params(insitu_schema) + if parameter_s and parameter_s not in insitu_params: raise NexusProcessingException( - reason="Parameter %s not supported. Must be one of 'sst', 'sss', 'wind'." % parameter_s, code=400) + reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400) try: start_time = request.get_start_datetime() @@ -262,10 +261,8 @@ class Matchup(NexusCalcSparkHandler): total_values = sum(len(v) for v in spark_result.values()) details = { "timeToComplete": int((end - start).total_seconds()), - "numInSituRecords": 0, - "numInSituMatched": total_values, - "numGriddedChecked": 0, - "numGriddedMatched": total_keys + "numSecondaryMatched": total_values, + "numPrimaryMatched": total_keys } matches = Matchup.convert_to_matches(spark_result) @@ -432,13 +429,8 @@ class DomsPoint(object): point.device = DomsPoint._variables_to_device(tile.variables) return point - insitu_schema = None - @staticmethod def from_edge_point(edge_point): - if DomsPoint.insitu_schema is None: - DomsPoint.insitu_schema = query_insitu_schema() - point = DomsPoint() x, y = edge_point['longitude'], edge_point['latitude'] @@ -513,7 +505,7 @@ class DomsPoint(object): val = edge_point.get(name) if not val: continue - unit = get_insitu_unit(name, DomsPoint.insitu_schema) + unit = get_insitu_unit(name, insitu_schema) data.append(DataPoint( variable_name=name, cf_variable_name=name, @@ -649,9 +641,22 @@ def add_meters_to_lon_lat(lon, lat, meters): return longitude, latitude +def get_insitu_params(insitu_schema): + """ + Get all possible insitu params from the CDMS insitu schema + """ + params = insitu_schema.get( + 'definitions', {}).get('observation', {}).get('properties', {}) + + # Filter params so only variables with units are considered + params = list(map( + lambda param: param[0], filter(lambda param: 'units'in param[1], params.items()))) + return params + + def get_insitu_unit(variable_name, insitu_schema): """ - Query the insitu API and retrieve the units for the given variable. + Retrieve the units from the insitu api schema endpoint for the given variable. If no units are available for this variable, return "None" """ properties = insitu_schema.get('definitions', {}).get('observation', {}).get('properties', {})
