This is an automated email from the ASF dual-hosted git repository. skperez pushed a commit to branch matchup-bug in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 33cda546ce78e3456c217ae076f8fcc0931e287e Author: skorper <[email protected]> AuthorDate: Wed Jun 15 02:25:10 2022 -0700 Fix matchup bugs --- .../webservice/algorithms/doms/DatasetListQuery.py | 25 ++++++++++++++-------- analysis/webservice/algorithms_spark/Matchup.py | 3 ++- analysis/webservice/apidocs/openapi.yml | 25 +++++++++++----------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/analysis/webservice/algorithms/doms/DatasetListQuery.py b/analysis/webservice/algorithms/doms/DatasetListQuery.py index 0d2e77f..ed0ca7e 100644 --- a/analysis/webservice/algorithms/doms/DatasetListQuery.py +++ b/analysis/webservice/algorithms/doms/DatasetListQuery.py @@ -98,15 +98,22 @@ class DomsDatasetListQueryHandler(BaseDomsHandler.BaseDomsQueryCalcHandler): for satellite in satellitesList: satellite["metadata"] = self.getMetadataForSource(satellite["shortName"]) - for insitu in config.ENDPOINTS: - depths, facets = self.getFacetsForInsituSource(insitu) - insituList.append({ - "name": insitu["name"], - "endpoint": insitu["url"], - "metadata": self.getMetadataForSource(insitu["name"]), - "depths": depths, - "facets": facets - }) + for insitu in config.INSITU_PROVIDER_MAP: + provider = insitu['name'] + for project in insitu['projects']: + project_name = project['name'] + insituList.append({ + 'name': project_name, + 'endpoint': f'{config.INSITU_API_ENDPOINT}?provider={provider}&project={project_name}', + 'metadata': { + 'Projects': [ + { + 'ShortName': project_name, + 'LongName': project_name + } + ] + } + }) values = { "satellite": satellitesList, diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py index cf20c5f..0f7e5ee 100644 --- a/analysis/webservice/algorithms_spark/Matchup.py +++ b/analysis/webservice/algorithms_spark/Matchup.py @@ -854,13 +854,14 @@ def query_edge(dataset, variable, startTime, endTime, bbox, platform, depth_min, # Get all edge results next_page_url = edge_endpoints.getEndpoint() while next_page_url is not None and next_page_url != 'NA': - logging.debug(f'Edge request {next_page_url}') + the_time = datetime.now() if session is not None: edge_page_request = session.get(next_page_url, params=params) else: edge_page_request = requests.get(next_page_url, params=params) edge_page_request.raise_for_status() + logging.info("%s Time to query edge %s", datetime.now() - the_time, edge_page_request.url) edge_page_response = json.loads(edge_page_request.text) diff --git a/analysis/webservice/apidocs/openapi.yml b/analysis/webservice/apidocs/openapi.yml index 2754463..ffe754e 100644 --- a/analysis/webservice/apidocs/openapi.yml +++ b/analysis/webservice/apidocs/openapi.yml @@ -14,10 +14,11 @@ tags: - name: Subsetting description: Data Subsetting API paths: - /match_spark_doms: + /match_spark: get: - summary: Execute matchup request + summary: Execute matchup request (AWS insitu) operationId: matchup + description: Endpoint to execute matchup request. If secondary dataset is insitu, use AWS insitu API. tags: - Matchup parameters: @@ -30,7 +31,7 @@ paths: schema: type: string x-dspopulate: - - satellite + - satellite example: avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020 - in: query name: secondary @@ -43,8 +44,8 @@ paths: schema: type: string x-dspopulate: - - satellite - - insitu + - satellite + - insitu example: icoads - in: query name: startTime @@ -126,7 +127,6 @@ paths: required: false schema: type: string - enum: ['sst', 'sss', 'wind'] default: sst example: sst - in: query @@ -174,10 +174,11 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' - /match_spark: + /match_spark_doms: get: - summary: Execute matchup request - operationId: matchup + summary: Execute matchup request (DOMS insitu) + operationId: matchup_doms + description: Endpoint to execute matchup request. If secondary dataset is insitu, use DOMS insitu API. tags: - Matchup parameters: @@ -190,7 +191,7 @@ paths: schema: type: string x-dspopulate: - - satellite + - satellite example: avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020 - in: query name: secondary @@ -202,9 +203,6 @@ paths: required: true schema: type: string - x-dspopulate: - - satellite - - insitu example: icoads - in: query name: startTime @@ -286,6 +284,7 @@ paths: required: false schema: type: string + enum: ['sst', 'sss', 'wind'] default: sst example: sst - in: query
