This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch ascending_latitudes in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 0d87acb96a23c71e8d42e9b127e003d7f1f3d90b Author: Elizabeth Yam <[email protected]> AuthorDate: Wed Nov 18 14:11:58 2020 -0800 SDAP-293: Automatic Kelvin-to-Celsius conversion (#22) Co-authored-by: Eamon Ford <[email protected]> --- .../services/CollectionProcessor.py | 1 + granule_ingester/granule_ingester/README.md | 11 +++++++++++ .../granule_ingester/processors/TileProcessor.py | 20 +++++++++++++++++++- .../granule_ingester/processors/kelvintocelsius.py | 18 +++++++++++------- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/collection_manager/collection_manager/services/CollectionProcessor.py b/collection_manager/collection_manager/services/CollectionProcessor.py index 96c461e..bab56fc 100644 --- a/collection_manager/collection_manager/services/CollectionProcessor.py +++ b/collection_manager/collection_manager/services/CollectionProcessor.py @@ -87,6 +87,7 @@ class CollectionProcessor: }, {'name': 'emptyTileFilter'}, {'name': 'subtract180FromLongitude'}, + {'name': 'kelvinToCelsius'}, { 'name': 'tileSummary', 'dataset_name': collection.dataset_id diff --git a/granule_ingester/granule_ingester/README.md b/granule_ingester/granule_ingester/README.md new file mode 100644 index 0000000..881461a --- /dev/null +++ b/granule_ingester/granule_ingester/README.md @@ -0,0 +1,11 @@ +## Plugin Processor Architecture +The operator can write a class that inherits from the `TileProcessor` class and implements the abstract function `process`, which among other things, takes in the NexusTile(`nexusproto.DataTile_pb2.NexusTile` object)) generated with default configurations and the NC4 Dataset(`xarray.Dataset` object), and allows the user to add further modifications to how granule data is saved. + +Any additional transformation the operator needs to accomplish must be done in this `process` method, which is what is ultimately called in the ingestion pipeline. Helper functions are suggested for breaking up complex procedures. + +The custom code file would be copied into the ingestion pods via the helm chart (see chart for local and mount paths). + +Example: `KelvinToCelsiusProcessor` +This processor checks the units of the saved variable. If it is some form of Kelvin, it automatically converts all of the temperature measurements to Celsius by subtracting 273.15 from each data point. The transformed data then replaces the default (untransformed) values and the processor returns the modified tile. + +#### TODO Add configuration option for unusual representations of temperature units. \ No newline at end of file diff --git a/granule_ingester/granule_ingester/processors/TileProcessor.py b/granule_ingester/granule_ingester/processors/TileProcessor.py index d62c504..47b1a2f 100644 --- a/granule_ingester/granule_ingester/processors/TileProcessor.py +++ b/granule_ingester/granule_ingester/processors/TileProcessor.py @@ -14,10 +14,28 @@ # limitations under the License. from abc import ABC, abstractmethod +from nexusproto.serialization import from_shaped_array, to_shaped_array +from nexusproto.DataTile_pb2 import NexusTile # TODO: make this an informal interface, not an abstract class class TileProcessor(ABC): @abstractmethod - def process(self, tile, *args, **kwargs): + def process(self, tile: NexusTile, *args, **kwargs): + # accessing the data + # the_tile_type = tile.tile.WhichOneof("tile_type") + # the_tile_data = getattr(tile.tile, the_tile_type) + + # get netCDF as xarray.Dataset object + # ds = kwargs['dataset'] + + # example transformation: + # var_data = from_shaped_array(the_tile_data.variable_data) - 273.15 + + # save transformed data back into tile + # the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data)) + + # return transformed tile + # return tile + pass diff --git a/granule_ingester/granule_ingester/processors/kelvintocelsius.py b/granule_ingester/granule_ingester/processors/kelvintocelsius.py index e728418..9ad4f49 100644 --- a/granule_ingester/granule_ingester/processors/kelvintocelsius.py +++ b/granule_ingester/granule_ingester/processors/kelvintocelsius.py @@ -13,19 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. - from nexusproto.serialization import from_shaped_array, to_shaped_array - +from nexusproto.DataTile_pb2 import NexusTile from granule_ingester.processors.TileProcessor import TileProcessor class KelvinToCelsius(TileProcessor): - def process(self, tile, *args, **kwargs): + def process(self, tile: NexusTile, *args, **kwargs): the_tile_type = tile.tile.WhichOneof("tile_type") the_tile_data = getattr(tile.tile, the_tile_type) - - var_data = from_shaped_array(the_tile_data.variable_data) - 273.15 - - the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data)) + kelvins = ['kelvin', 'degk', 'deg_k', 'degreesk', 'degrees_k', 'degree_k', 'degreek'] + + if 'dataset' in kwargs: + ds = kwargs['dataset'] + variable_name = tile.summary.data_var_name + variable_unit = ds.variables[variable_name].attrs['units'] + if any([unit in variable_unit.lower() for unit in kelvins]): + var_data = from_shaped_array(the_tile_data.variable_data) - 273.15 + the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data)) return tile
