This is an automated email from the ASF dual-hosted git repository. fgreg pushed a commit to branch v1.0.0-rc1 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ningesterpy.git
commit df5e890f35e5c3450c3c1feac7c4ab21775b3fd9 Author: Frank Greguska <[email protected]> AuthorDate: Tue Mar 6 16:11:01 2018 -0800 new python processor which can promote a variable into a global attribute --- sdap/processors/__init__.py | 2 + .../processors/promotevariabletoglobalattribute.py | 49 +++++++++++++++++++++ tests/datafiles/not_empty_wswm.nc | Bin 33119 -> 1041568 bytes tests/processorchain_test.py | 33 ++++++++++++++ tests/promotevariabletoglobalattribute_test.py | 38 ++++++++++++++++ 5 files changed, 122 insertions(+) diff --git a/sdap/processors/__init__.py b/sdap/processors/__init__.py index 8a48500..6d4a679 100644 --- a/sdap/processors/__init__.py +++ b/sdap/processors/__init__.py @@ -56,6 +56,7 @@ from sdap.processors.deleteunitaxis import DeleteUnitAxis from sdap.processors.emptytilefilter import EmptyTileFilter from sdap.processors.kelvintocelsius import KelvinToCelsius from sdap.processors.normalizetimebeginningofmonth import NormalizeTimeBeginningOfMonth +from sdap.processors.promotevariabletoglobalattribute import PromoteVariableToGlobalAttribute from sdap.processors.regrid1x1 import Regrid1x1 from sdap.processors.subtract180longitude import Subtract180Longitude from sdap.processors.tilereadingprocessor import GridReadingProcessor, SwathReadingProcessor, TimeSeriesReadingProcessor @@ -70,6 +71,7 @@ INSTALLED_PROCESSORS = { "EmptyTileFilter": EmptyTileFilter, "KelvinToCelsius": KelvinToCelsius, "NormalizeTimeBeginningOfMonth": NormalizeTimeBeginningOfMonth, + "PromoteVariableToGlobalAttribute": PromoteVariableToGlobalAttribute, "Regrid1x1": Regrid1x1, "Subtract180Longitude": Subtract180Longitude, "GridReadingProcessor": GridReadingProcessor, diff --git a/sdap/processors/promotevariabletoglobalattribute.py b/sdap/processors/promotevariabletoglobalattribute.py new file mode 100644 index 0000000..11fa9d8 --- /dev/null +++ b/sdap/processors/promotevariabletoglobalattribute.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nexusproto.DataTile_pb2 +from netCDF4 import Dataset + +from sdap.processors import NexusTileProcessor + + +class PromoteVariableToGlobalAttribute(NexusTileProcessor): + + def __init__(self, attribute_name, variable_name, dimensioned_by, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.attribute_name = attribute_name + self.variable_name = variable_name + self.dimensioned_by = dimensioned_by + + def process_nexus_tile(self, nexus_tile): + output_tile = nexusproto.DataTile_pb2.NexusTile() + output_tile.CopyFrom(nexus_tile) + + file_path = output_tile.summary.granule + file_path = file_path[len('file:'):] if file_path.startswith('file:') else file_path + + dimtoslice = {} + for dimension in output_tile.summary.section_spec.split(','): + name, start, stop = dimension.split(':') + dimtoslice[name] = slice(int(start), int(stop)) + + with Dataset(file_path) as ds: + new_attr = output_tile.summary.global_attributes.add() + new_attr.name = self.attribute_name + new_attr.values.extend( + [str(v) for v in ds[self.variable_name][[dimtoslice[dim] for dim in self.dimensioned_by]]]) + + yield output_tile diff --git a/tests/datafiles/not_empty_wswm.nc b/tests/datafiles/not_empty_wswm.nc index 772bbcb..ce0ebcc 100644 Binary files a/tests/datafiles/not_empty_wswm.nc and b/tests/datafiles/not_empty_wswm.nc differ diff --git a/tests/processorchain_test.py b/tests/processorchain_test.py index 7657ba6..d7b5dfa 100644 --- a/tests/processorchain_test.py +++ b/tests/processorchain_test.py @@ -130,6 +130,39 @@ class TestRunChainMethod(unittest.TestCase): self.assertEqual(0, len(results)) + def test_run_chain_promote_var(self): + processor_list = [ + {'name': 'GridReadingProcessor', + 'config': {'latitude': 'lat', + 'longitude': 'lon', + 'time': 'time', + 'variable_to_read': 'analysed_sst'}}, + {'name': 'EmptyTileFilter'}, + {'name': 'KelvinToCelsius'}, + {'name': 'PromoteVariableToGlobalAttribute', + 'config': { + 'attribute_name': 'time_i', + 'variable_name': 'time', + 'dimensioned_by': ['time'] + }}, + {'name': 'TileSummarizingProcessor'} + ] + processorchain = ProcessorChain(processor_list) + + test_file = path.join(path.dirname(__file__), 'datafiles', 'partial_empty_mur.nc4') + + input_tile = nexusproto.NexusTile() + tile_summary = nexusproto.TileSummary() + tile_summary.granule = "file:%s" % test_file + tile_summary.section_spec = "time:0:1,lat:489:499,lon:0:10" + input_tile.summary.CopyFrom(tile_summary) + + results = list(processorchain.process(input_tile)) + + self.assertEqual(1, len(results)) + tile = results[0] + self.assertEqual("1104483600", tile.summary.global_attributes[0].values[0]) + if __name__ == '__main__': unittest.main() diff --git a/tests/promotevariabletoglobalattribute_test.py b/tests/promotevariabletoglobalattribute_test.py new file mode 100644 index 0000000..1fbe21d --- /dev/null +++ b/tests/promotevariabletoglobalattribute_test.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from os import path + +import sdap.processors +import nexusproto.DataTile_pb2 + + +class TestReadWSWMData(unittest.TestCase): + + def test_read_not_empty_wswm(self): + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_wswm.nc') + + promoter = sdap.processors.PromoteVariableToGlobalAttribute('rivid_i', 'rivid', ('rivid',)) + + input_tile = nexusproto.DataTile_pb2.NexusTile() + tile_summary = nexusproto.DataTile_pb2.TileSummary() + tile_summary.granule = "file:%s" % test_file + tile_summary.section_spec = "time:0:5832,rivid:0:1" + input_tile.summary.CopyFrom(tile_summary) + + results = list(promoter.process(input_tile)) + + print(results) \ No newline at end of file
