This is an automated email from the ASF dual-hosted git repository.
nchung pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/master by this push:
new 129d41d CDMS-50: Updated matchup algorithm to remove parameter
restriction (#126)
129d41d is described below
commit 129d41d778e222c4f9b2cd6bcf4ff6626ad1aa64
Author: Stepheny Perez <[email protected]>
AuthorDate: Fri Apr 16 16:45:57 2021 -0700
CDMS-50: Updated matchup algorithm to remove parameter restriction (#126)
* CDMS-50: Updated matchup algorithm to remove parameter restriction
* CDMS-50: Doms point uses sat var name
* CDMS-50: Cleanup
---
analysis/webservice/algorithms_spark/Matchup.py | 75 ++++++++++++++-----------
data-access/nexustiles/model/nexusmodel.py | 1 +
data-access/nexustiles/nexustiles.py | 5 ++
helm/values.yaml | 11 ++--
4 files changed, 53 insertions(+), 39 deletions(-)
diff --git a/analysis/webservice/algorithms_spark/Matchup.py
b/analysis/webservice/algorithms_spark/Matchup.py
index 93ab4ee..0734965 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -69,7 +69,7 @@ class Matchup(NexusCalcSparkHandler):
"parameter": {
"name": "Match-Up Parameter",
"type": "string",
- "description": "The parameter of interest used for the match up.
One of 'sst', 'sss', 'wind'. Required"
+ "description": "The parameter of interest used for the match up.
One of 'sst', 'sss', 'wind'. Optional"
},
"startTime": {
"name": "Start Time",
@@ -152,8 +152,8 @@ class Matchup(NexusCalcSparkHandler):
if matchup_ds_names is None:
raise NexusProcessingException(reason="'matchup' argument is
required", code=400)
- parameter_s = request.get_argument('parameter', 'sst')
- if parameter_s not in ['sst', 'sss', 'wind']:
+ parameter_s = request.get_argument('parameter')
+ if parameter_s and parameter_s not in ['sst', 'sss', 'wind']:
raise NexusProcessingException(
reason="Parameter %s not supported. Must be one of 'sst',
'sss', 'wind'." % parameter_s, code=400)
@@ -302,7 +302,7 @@ class Matchup(NexusCalcSparkHandler):
@staticmethod
def domspoint_to_dict(domspoint):
- return {
+ doms_dict = {
"sea_water_temperature": domspoint.sst,
"sea_water_temperature_depth": domspoint.sst_depth,
"sea_water_salinity": domspoint.sss,
@@ -321,6 +321,9 @@ class Matchup(NexusCalcSparkHandler):
"id": domspoint.data_id,
"source": domspoint.source,
}
+ if domspoint.satellite_var_name:
+ doms_dict[domspoint.satellite_var_name] =
domspoint.satellite_var_value
+ return doms_dict
class DomsPoint(object):
@@ -332,6 +335,9 @@ class DomsPoint(object):
self.depth = depth
self.data_id = data_id
+ self.satellite_var_name = None
+ self.satellite_var_value = None
+
self.wind_u = None
self.wind_v = None
self.wind_direction = None
@@ -350,32 +356,28 @@ class DomsPoint(object):
return str(self.__dict__)
@staticmethod
- def from_nexus_point(nexus_point, tile=None, parameter='sst'):
+ def from_nexus_point(nexus_point, tile=None):
point = DomsPoint()
point.data_id = "%s[%s]" % (tile.tile_id, nexus_point.index)
- # TODO Not an ideal solution; but it works for now.
- if parameter == 'sst':
- point.sst = nexus_point.data_val.item()
- elif parameter == 'sss':
- point.sss = nexus_point.data_val.item()
- elif parameter == 'wind':
- point.wind_u = nexus_point.data_val.item()
- try:
- point.wind_v =
tile.meta_data['wind_v'][tuple(nexus_point.index)].item()
- except (KeyError, IndexError):
- pass
- try:
- point.wind_direction =
tile.meta_data['wind_dir'][tuple(nexus_point.index)].item()
- except (KeyError, IndexError):
- pass
- try:
- point.wind_speed =
tile.meta_data['wind_speed'][tuple(nexus_point.index)].item()
- except (KeyError, IndexError):
- pass
- else:
- raise NotImplementedError('%s not supported. Only sst, sss, and
wind parameters are supported.' % parameter)
+ # Get the name of the satellite variable from the source NetCDF
+ satellite_var_name = tile.var_name
+ point.satellite_var_name = satellite_var_name
+ point.satellite_var_value = nexus_point.data_val.item()
+
+ try:
+ point.wind_v =
tile.meta_data['wind_v'][tuple(nexus_point.index)].item()
+ except (KeyError, IndexError):
+ pass
+ try:
+ point.wind_direction =
tile.meta_data['wind_dir'][tuple(nexus_point.index)].item()
+ except (KeyError, IndexError):
+ pass
+ try:
+ point.wind_speed =
tile.meta_data['wind_speed'][tuple(nexus_point.index)].item()
+ except (KeyError, IndexError):
+ pass
point.longitude = nexus_point.longitude.item()
point.latitude = nexus_point.latitude.item()
@@ -642,11 +644,16 @@ def match_tile_to_point_generator(tile_service, tile_id,
m_tree, edge_results, s
print("%s Time to query primary tree for tile %s" % (str(datetime.now() -
a_time), tile_id))
for i, point_matches in enumerate(matched_indexes):
if len(point_matches) > 0:
- p_nexus_point = NexusPoint(tile.latitudes[valid_indices[i][1]],
- tile.longitudes[valid_indices[i][2]],
None,
- tile.times[valid_indices[i][0]],
valid_indices[i],
- tile.data[tuple(valid_indices[i])])
- p_doms_point = DomsPoint.from_nexus_point(p_nexus_point,
tile=tile, parameter=search_parameter)
+ p_nexus_point = NexusPoint(
+ latitude=tile.latitudes[valid_indices[i][1]],
+ longitude=tile.longitudes[valid_indices[i][2]],
+ depth=None,
+ time=tile.times[valid_indices[i][0]],
+ index=valid_indices[i],
+ data_val=tile.data[tuple(valid_indices[i])]
+ )
+
+ p_doms_point = DomsPoint.from_nexus_point(p_nexus_point, tile=tile)
for m_point_index in point_matches:
m_doms_point =
DomsPoint.from_edge_point(edge_results[m_point_index])
yield p_doms_point, m_doms_point
@@ -672,8 +679,7 @@ def query_edge(dataset, variable, startTime, endTime, bbox,
platform, depth_min,
# Assume we were passed a list
pass
- params = {"variable": variable,
- "startTime": startTime,
+ params = {"startTime": startTime,
"endTime": endTime,
"bbox": bbox,
"minDepth": depth_min,
@@ -681,6 +687,9 @@ def query_edge(dataset, variable, startTime, endTime, bbox,
platform, depth_min,
"platform": platform,
"itemsPerPage": itemsPerPage, "startIndex": startIndex, "stats":
str(stats).lower()}
+ if variable is not None:
+ params["variable"] = variable
+
dataset_url = edge_endpoints.getEndpointByName(dataset)['url']
if session is not None:
edge_request = session.get(dataset_url, params=params)
diff --git a/data-access/nexustiles/model/nexusmodel.py
b/data-access/nexustiles/model/nexusmodel.py
index 2996448..01b4283 100644
--- a/data-access/nexustiles/model/nexusmodel.py
+++ b/data-access/nexustiles/model/nexusmodel.py
@@ -36,6 +36,7 @@ class Tile(object):
self.max_time = None
self.tile_stats = None
+ self.var_name = None
self.latitudes = None # This should be a 1-d ndarray
self.longitudes = None # This should be a 1-d ndarray
diff --git a/data-access/nexustiles/nexustiles.py
b/data-access/nexustiles/nexustiles.py
index 64361e3..eb847d2 100644
--- a/data-access/nexustiles/nexustiles.py
+++ b/data-access/nexustiles/nexustiles.py
@@ -518,6 +518,11 @@ class NexusTileService(object):
except KeyError:
pass
+ try:
+ tile.var_name = solr_doc['tile_var_name_s']
+ except KeyError:
+ pass
+
tiles.append(tile)
return tiles
diff --git a/helm/values.yaml b/helm/values.yaml
index aba22e6..a2cba8c 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -6,8 +6,7 @@ rootWebpage:
webapp:
enabled: true
distributed:
- image: nexusjpl/nexus-webapp:distributed.0.4.0
-
+ image: nexusjpl/nexus-webapp:distributed.0.4.1
## Use any of the driver configuration options available at
##
https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md
driver:
@@ -68,11 +67,11 @@ ingestion:
## path is the path on either local filesystem or NFS mount at which
## the granule files are stored. This will be ignored if S3 ingestion is
enabled.
- path:
+ path:
s3:
## If bucket has a value, S3 ingestion will be enabled (and nfsServer
will be ignored even if it has a value).
- bucket:
+ bucket:
## awsCredsEnvs can include any environment variables that contain AWS
credentials
awsCredsEnvs: {}
@@ -85,7 +84,7 @@ ingestion:
## Name of a ConfigMap containing the Collections Config YAML.
## Leave this blank if Git is enabled below.
- configMap:
+ configMap:
## Load the Collections Config file from a git repository.
git:
@@ -107,7 +106,7 @@ ingestion:
## The values in this section are relevant if using Solr, Zookeeper, or
Cassandra that were not deployed from this Helm chart
external:
- solrHostAndPort:
+ solrHostAndPort:
zookeeperHostAndPort:
cassandraHost:
cassandraUsername:
