This is an automated email from the ASF dual-hosted git repository.
nchung pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new bc94d29 SDAP-347 Added day_of_year_i to Solr and Elasticsearch
writers, and fixed issue in the way we access units in kelvintocelsius
processor (#42)
bc94d29 is described below
commit bc94d2923cdea7a98eaf9e5b06dae11bc88f8594
Author: WicketWarrick <[email protected]>
AuthorDate: Thu Sep 23 02:11:39 2021 +0200
SDAP-347 Added day_of_year_i to Solr and Elasticsearch writers, and fixed
issue in the way we access units in kelvintocelsius processor (#42)
Co-authored-by: Dorian FOUQUIER <[email protected]>
Co-authored-by: Nga Chung <[email protected]>
---
.../granule_ingester/processors/kelvintocelsius.py | 14 ++++++++------
.../granule_ingester/writers/ElasticsearchStore.py | 3 ++-
granule_ingester/granule_ingester/writers/SolrStore.py | 3 ++-
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/granule_ingester/granule_ingester/processors/kelvintocelsius.py
b/granule_ingester/granule_ingester/processors/kelvintocelsius.py
index 755f3be..d1d362e 100644
--- a/granule_ingester/granule_ingester/processors/kelvintocelsius.py
+++ b/granule_ingester/granule_ingester/processors/kelvintocelsius.py
@@ -33,12 +33,13 @@ class KelvinToCelsius(TileProcessor):
for each in copied_variable_name:
try:
logger.info(f'for ds.variables[each].attrs :
{ds.variables[each].attrs}')
- if 'units' in ds.variables[each].attrs:
- variable_unit.extend(ds.variables[each].attrs['units'])
- elif 'Units' in ds.variables[each].attrs:
- variable_unit.extend(ds.variables[each].attrs['Units'])
- elif 'UNITS' in ds.variables[each].attrs:
- variable_unit.extend(ds.variables[each].attrs['UNITS'])
+ for unit_attr in ('units', 'Units', 'UNITS'):
+ if unit_attr in ds.variables[each].attrs:
+ if isinstance(ds.variables[each].attrs[unit_attr],
list):
+
variable_unit.extend(ds.variables[each].attrs[unit_attr])
+ else:
+
variable_unit.append(ds.variables[each].attrs[unit_attr])
+ break
except Exception as e:
logger.exception(f'some error in __retrieve_var_units:
{str(e)}')
return variable_unit
@@ -62,4 +63,5 @@ class KelvinToCelsius(TileProcessor):
if any([unit in variable_unit for unit in kelvins]):
var_data = from_shaped_array(the_tile_data.variable_data) -
273.15
the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
+
return tile
diff --git a/granule_ingester/granule_ingester/writers/ElasticsearchStore.py
b/granule_ingester/granule_ingester/writers/ElasticsearchStore.py
index 83b1ed9..b18b566 100644
--- a/granule_ingester/granule_ingester/writers/ElasticsearchStore.py
+++ b/granule_ingester/granule_ingester/writers/ElasticsearchStore.py
@@ -63,7 +63,7 @@ class ElasticsearchStore(MetadataStore):
min_time =
datetime.strftime(datetime.utcfromtimestamp(stats.min_time), self.iso)
max_time =
datetime.strftime(datetime.utcfromtimestamp(stats.max_time), self.iso)
-
+ day_of_year =
datetime.utcfromtimestamp(stats.min_time).timetuple().tm_yday
geo = self.determine_geo(bbox)
granule_file_name: str = Path(summary.granule).name # get base
filename
@@ -82,6 +82,7 @@ class ElasticsearchStore(MetadataStore):
'dataset_s': summary.dataset_name,
'granule_s': granule_file_name,
'tile_var_name_s': var_name,
+ 'day_of_year_i': day_of_year,
'tile_min_lon': round(bbox.lon_min, 3),
'tile_max_lon': round(bbox.lon_max, 3),
'tile_min_lat': round(bbox.lat_min, 3),
diff --git a/granule_ingester/granule_ingester/writers/SolrStore.py
b/granule_ingester/granule_ingester/writers/SolrStore.py
index 2e873d7..def70c5 100644
--- a/granule_ingester/granule_ingester/writers/SolrStore.py
+++ b/granule_ingester/granule_ingester/writers/SolrStore.py
@@ -97,7 +97,7 @@ class SolrStore(MetadataStore):
min_time =
datetime.strftime(datetime.utcfromtimestamp(stats.min_time), self.iso)
max_time =
datetime.strftime(datetime.utcfromtimestamp(stats.max_time), self.iso)
-
+ day_of_year =
datetime.utcfromtimestamp(stats.min_time).timetuple().tm_yday
geo = self.determine_geo(bbox)
granule_file_name: str = Path(summary.granule).name # get base
filename
@@ -123,6 +123,7 @@ class SolrStore(MetadataStore):
'dataset_s': summary.dataset_name,
'granule_s': granule_file_name,
'tile_var_name_ss': var_names,
+ 'day_of_year_i': day_of_year,
'tile_min_lon': bbox.lon_min,
'tile_max_lon': bbox.lon_max,
'tile_min_lat': bbox.lat_min,
