This is an automated email from the ASF dual-hosted git repository.
nchung pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/master by this push:
new 95f92fe SDAP-355: Update matchup parameter and output to use primary
and secondary (#142)
95f92fe is described below
commit 95f92fe245853e8a1e0e5afe695d3eea542cae25
Author: Stepheny Perez <[email protected]>
AuthorDate: Fri Oct 29 16:55:57 2021 -0700
SDAP-355: Update matchup parameter and output to use primary and secondary
(#142)
* renamed 'matchup' terminology to 'secondary'
* Updated tests to work with 'matchup' -> 'secondary' terminology change
* Updated OpenAPI spec
* Remove circular reference in openapi.yml
---
analysis/tests/algorithms_spark/test_matchup.py | 18 ++++----
analysis/webservice/algorithms_spark/Matchup.py | 38 +++++++--------
analysis/webservice/apidocs/openapi.yml | 61 +++++++++++++++----------
3 files changed, 64 insertions(+), 53 deletions(-)
diff --git a/analysis/tests/algorithms_spark/test_matchup.py
b/analysis/tests/algorithms_spark/test_matchup.py
index f0f9c0d..db35809 100644
--- a/analysis/tests/algorithms_spark/test_matchup.py
+++ b/analysis/tests/algorithms_spark/test_matchup.py
@@ -63,7 +63,7 @@ def test_matchup_args():
tile_ids = [1]
polygon_wkt = 'POLYGON((-34.98 29.54, -30.1 29.54, -30.1 31.00, -34.98
31.00, -34.98 29.54))'
primary_ds_name = 'primary-ds-name'
- matchup_ds_names = 'test'
+ secondary_ds_names = 'test'
parameter = 'sst'
depth_min = 0.0
depth_max = 1.0
@@ -74,7 +74,7 @@ def test_matchup_args():
yield dict(
tile_ids=tile_ids,
primary_b=MockSparkParam(primary_ds_name),
- matchup_b=MockSparkParam(matchup_ds_names),
+ secondary_b=MockSparkParam(secondary_ds_names),
parameter_b=MockSparkParam(parameter),
tt_b=MockSparkParam(time_tolerance),
rt_b=MockSparkParam(radius_tolerance),
@@ -238,12 +238,12 @@ def test_calc(test_matchup_args):
assert matches['x'] == '-180'
assert matches['y'] == '-90'
- assert json_matchup_result['data'][0]['data'][0]['variable_value'] ==
10.0
- assert json_matchup_result['data'][1]['data'][0]['variable_value'] ==
20.0
- assert
json_matchup_result['data'][0]['matches'][0]['data'][0]['variable_value'] ==
30.0
- assert
json_matchup_result['data'][0]['matches'][1]['data'][0]['variable_value'] ==
40.0
- assert
json_matchup_result['data'][1]['matches'][0]['data'][0]['variable_value'] ==
30.0
- assert
json_matchup_result['data'][1]['matches'][1]['data'][0]['variable_value'] ==
40.0
+ assert json_matchup_result['data'][0]['primary'][0]['variable_value']
== 10.0
+ assert json_matchup_result['data'][1]['primary'][0]['variable_value']
== 20.0
+ assert
json_matchup_result['data'][0]['matches'][0]['secondary'][0]['variable_value']
== 30.0
+ assert
json_matchup_result['data'][0]['matches'][1]['secondary'][0]['variable_value']
== 40.0
+ assert
json_matchup_result['data'][1]['matches'][0]['secondary'][0]['variable_value']
== 30.0
+ assert
json_matchup_result['data'][1]['matches'][1]['secondary'][0]['variable_value']
== 40.0
assert json_matchup_result['details']['numInSituMatched'] == 4
assert json_matchup_result['details']['numGriddedMatched'] == 2
@@ -315,7 +315,7 @@ def test_match_satellite_to_insitu(test_dir, test_tile,
test_matchup_args):
match_args = dict(
tile_ids=tile_ids,
primary_b=MockSparkParam(primary_ds_name),
- matchup_b=MockSparkParam(matchup_ds_names),
+ secondary_b=MockSparkParam(matchup_ds_names),
parameter_b=MockSparkParam(parameter),
tt_b=MockSparkParam(time_tolerance),
rt_b=MockSparkParam(radius_tolerance),
diff --git a/analysis/webservice/algorithms_spark/Matchup.py
b/analysis/webservice/algorithms_spark/Matchup.py
index de9551b..66ef2a9 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -63,7 +63,7 @@ class Matchup(NexusCalcSparkHandler):
"type": "string",
"description": "The Primary dataset used to find matches for.
Required"
},
- "matchup": {
+ "secondary": {
"name": "Match-Up Datasets",
"type": "comma-delimited string",
"description": "The Dataset(s) being searched for measurements
that match the Primary. Required"
@@ -150,9 +150,9 @@ class Matchup(NexusCalcSparkHandler):
primary_ds_name = request.get_argument('primary', None)
if primary_ds_name is None:
raise NexusProcessingException(reason="'primary' argument is
required", code=400)
- matchup_ds_names = request.get_argument('matchup', None)
- if matchup_ds_names is None:
- raise NexusProcessingException(reason="'matchup' argument is
required", code=400)
+ secondary_ds_names = request.get_argument('secondary', None)
+ if secondary_ds_names is None:
+ raise NexusProcessingException(reason="'secondary' argument is
required", code=400)
parameter_s = request.get_argument('parameter')
if parameter_s and parameter_s not in ['sst', 'sss', 'wind']:
@@ -204,7 +204,7 @@ class Matchup(NexusCalcSparkHandler):
start_seconds_from_epoch = int((start_time - EPOCH).total_seconds())
end_seconds_from_epoch = int((end_time - EPOCH).total_seconds())
- return bounding_polygon, primary_ds_name, matchup_ds_names,
parameter_s, \
+ return bounding_polygon, primary_ds_name, secondary_ds_names,
parameter_s, \
start_time, start_seconds_from_epoch, end_time,
end_seconds_from_epoch, \
depth_min, depth_max, time_tolerance, radius_tolerance, \
platforms, match_once, result_size_limit
@@ -212,7 +212,7 @@ class Matchup(NexusCalcSparkHandler):
def calc(self, request, **args):
start = datetime.utcnow()
# TODO Assuming Satellite primary
- bounding_polygon, primary_ds_name, matchup_ds_names, parameter_s, \
+ bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
start_time, start_seconds_from_epoch, end_time,
end_seconds_from_epoch, \
depth_min, depth_max, time_tolerance, radius_tolerance, \
platforms, match_once, result_size_limit =
self.parse_arguments(request)
@@ -235,7 +235,7 @@ class Matchup(NexusCalcSparkHandler):
self.log.debug("Calling Spark Driver")
try:
spark_result = spark_matchup_driver(tile_ids,
wkt.dumps(bounding_polygon), primary_ds_name,
- matchup_ds_names, parameter_s,
depth_min, depth_max, time_tolerance,
+ secondary_ds_names,
parameter_s, depth_min, depth_max, time_tolerance,
radius_tolerance, platforms,
match_once, self.tile_service_factory, sc=self._sc)
except Exception as e:
self.log.exception(e)
@@ -246,7 +246,7 @@ class Matchup(NexusCalcSparkHandler):
self.log.debug("Building and saving results")
args = {
"primary": primary_ds_name,
- "matchup": matchup_ds_names,
+ "matchup": secondary_ds_names,
"startTime": start_time,
"endTime": end_time,
"bbox": request.get_argument('b'),
@@ -295,15 +295,15 @@ class Matchup(NexusCalcSparkHandler):
def convert_to_matches(cls, spark_result):
matches = []
for primary_domspoint, matched_domspoints in spark_result.items():
- p_matched = [cls.domspoint_to_dict(p_match) for p_match in
matched_domspoints]
+ p_matched = [cls.domspoint_to_dict(p_match, 'secondary') for
p_match in matched_domspoints]
- primary = cls.domspoint_to_dict(primary_domspoint)
+ primary = cls.domspoint_to_dict(primary_domspoint, 'primary')
primary['matches'] = list(p_matched)
matches.append(primary)
return matches
@staticmethod
- def domspoint_to_dict(domspoint):
+ def domspoint_to_dict(domspoint, data_key_name='data'):
doms_dict = {
"platform": doms_values.getPlatformById(domspoint.platform),
"device": doms_values.getDeviceById(domspoint.device),
@@ -314,7 +314,7 @@ class Matchup(NexusCalcSparkHandler):
"fileurl": domspoint.file_url,
"id": domspoint.data_id,
"source": domspoint.source,
- "data": [data_point.__dict__ for data_point in domspoint.data]
+ data_key_name: [data_point.__dict__ for data_point in
domspoint.data]
}
return doms_dict
@@ -480,14 +480,14 @@ from threading import Lock
DRIVER_LOCK = Lock()
-def spark_matchup_driver(tile_ids, bounding_wkt, primary_ds_name,
matchup_ds_names, parameter, depth_min, depth_max,
+def spark_matchup_driver(tile_ids, bounding_wkt, primary_ds_name,
secondary_ds_names, parameter, depth_min, depth_max,
time_tolerance, radius_tolerance, platforms,
match_once, tile_service_factory, sc=None):
from functools import partial
with DRIVER_LOCK:
# Broadcast parameters
primary_b = sc.broadcast(primary_ds_name)
- matchup_b = sc.broadcast(matchup_ds_names)
+ secondary_b = sc.broadcast(secondary_ds_names)
depth_min_b = sc.broadcast(float(depth_min) if depth_min is not None
else None)
depth_max_b = sc.broadcast(float(depth_max) if depth_max is not None
else None)
tt_b = sc.broadcast(time_tolerance)
@@ -504,7 +504,7 @@ def spark_matchup_driver(tile_ids, bounding_wkt,
primary_ds_name, matchup_ds_nam
partial(
match_satellite_to_insitu,
primary_b=primary_b,
- matchup_b=matchup_b,
+ secondary_b=secondary_b,
parameter_b=parameter_b,
tt_b=tt_b,
rt_b=rt_b,
@@ -599,7 +599,7 @@ def tile_to_edge_points(tile):
return edge_points
-def match_satellite_to_insitu(tile_ids, primary_b, matchup_b, parameter_b,
tt_b, rt_b, platforms_b,
+def match_satellite_to_insitu(tile_ids, primary_b, secondary_b, parameter_b,
tt_b, rt_b, platforms_b,
bounding_wkt_b, depth_min_b, depth_max_b,
tile_service_factory):
the_time = datetime.now()
tile_ids = list(tile_ids)
@@ -636,14 +636,14 @@ def match_satellite_to_insitu(tile_ids, primary_b,
matchup_b, parameter_b, tt_b,
str(datetime.now() - the_time), tile_ids[0], tile_ids[-1]))
# Query edge for all points within the spatial-temporal extents of this
partition
- is_insitu_dataset = edge_endpoints.getEndpointByName(matchup_b.value)
+ is_insitu_dataset = edge_endpoints.getEndpointByName(secondary_b.value)
if is_insitu_dataset:
the_time = datetime.now()
edge_session = requests.Session()
edge_results = []
with edge_session:
- for insitudata_name in matchup_b.value.split(','):
+ for insitudata_name in secondary_b.value.split(','):
bbox = ','.join(
[str(matchup_min_lon), str(matchup_min_lat),
str(matchup_max_lon), str(matchup_max_lat)])
edge_response = query_edge(insitudata_name, parameter_b.value,
matchup_min_time, matchup_max_time, bbox,
@@ -682,7 +682,7 @@ def match_satellite_to_insitu(tile_ids, primary_b,
matchup_b, parameter_b, tt_b,
matchup_tiles = tile_service.find_tiles_in_polygon(
bounding_polygon=polygon,
- ds=matchup_b.value,
+ ds=secondary_b.value,
start_time=matchup_min_time,
end_time=matchup_max_time,
fetch_data=True,
diff --git a/analysis/webservice/apidocs/openapi.yml
b/analysis/webservice/apidocs/openapi.yml
index 0317cf5..773814f 100644
--- a/analysis/webservice/apidocs/openapi.yml
+++ b/analysis/webservice/apidocs/openapi.yml
@@ -31,7 +31,7 @@ paths:
type: string
example:
avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020
- in: query
- name: matchup
+ name: secondary
description: |
The dataset(s) being searched for measurements that match
the measurements in primary. One or more (comma-separated)
@@ -632,26 +632,34 @@ components:
data:
type: array
items:
- $ref: '#/components/schemas/DomsPoint'
- DomsPoint:
+ $ref: '#/components/schemas/DomsPointPrimary'
+ DomsPointPrimary:
+ allOf:
+ - $ref: '#/components/schemas/DomsPointBase'
+ - type: object
+ properties:
+ matches:
+ type: array
+ items:
+ $ref: '#/components/schemas/DomsPointSecondary'
+ - type: object
+ properties:
+ primary:
+ type: array
+ items:
+ $ref: '#/components/schemas/DomsDataPoint'
+ DomsPointSecondary:
+ allOf:
+ - $ref: '#/components/schemas/DomsPointBase'
+ - type: object
+ properties:
+ secondary:
+ type: array
+ items:
+ $ref: '#/components/schemas/DomsDataPoint'
+ DomsPointBase:
type: object
properties:
- sea_water_temperature:
- type: number
- sea_water_temperature_depth:
- type: number
- sea_water_salinity:
- type: number
- sea_water_salinity_depth:
- type: number
- wind_speed:
- type: number
- wind_direction:
- type: number
- wind_u:
- type: number
- wind_v:
- type: number
platform:
type: string
device:
@@ -671,12 +679,15 @@ components:
type: string
source:
type: string
- analysed_sst:
- type: number # TODO Fix this
- matches:
- type: array
- items:
- $ref: '#/components/schemas/DomsPoint'
+ DomsDataPoint:
+ type: object
+ properties:
+ variable_name:
+ type: string
+ cf_variable_name:
+ type: string
+ variable_value:
+ type: number
DomsList:
allOf:
- $ref: '#/components/schemas/DomsQueryResult'
