initial commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/commit/df17d945 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/tree/df17d945 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/diff/df17d945 Branch: refs/heads/master Commit: df17d9455f909afb4fb6b4b20215b0241b448cd7 Parents: Author: Frank Greguska <[email protected]> Authored: Wed Dec 27 16:29:08 2017 -0600 Committer: Frank Greguska <[email protected]> Committed: Wed Dec 27 16:29:08 2017 -0600 ---------------------------------------------------------------------- .gitignore | 9 + ningesterpy/__init__.py | 5 + ningesterpy/ningesterpy.py | 80 +++++ ningesterpy/processors/__init__.py | 66 ++++ ningesterpy/processors/callncpdq.py | 48 +++ ningesterpy/processors/callncra.py | 48 +++ ningesterpy/processors/computespeeddirfromuv.py | 68 ++++ ningesterpy/processors/emptytilefilter.py | 31 ++ ningesterpy/processors/kelvintocelsius.py | 21 ++ .../processors/normalizetimebeginningofmonth.py | 30 ++ ningesterpy/processors/processorchain.py | 72 ++++ ningesterpy/processors/regrid1x1.py | 136 ++++++++ ningesterpy/processors/subtract180longitude.py | 32 ++ ningesterpy/processors/tilereadingprocessor.py | 266 +++++++++++++++ .../processors/tilesummarizingprocessor.py | 93 ++++++ ningesterpy/processors/winddirspeedtouv.py | 90 +++++ requirements.txt | 12 + setup.py | 29 ++ tests/__init__.py | 4 + tests/callncpdq_test.py | 53 +++ tests/computespeeddirfromuv_test.py | 114 +++++++ tests/convert_iceshelf.py | 78 +++++ tests/datafiles/empty_mur.nc4 | Bin 0 -> 60937 bytes tests/datafiles/not_empty_ascatb.nc4 | Bin 0 -> 78036 bytes tests/datafiles/not_empty_avhrr.nc4 | Bin 0 -> 49511 bytes tests/datafiles/not_empty_ccmp.nc | Bin 0 -> 206870 bytes tests/datafiles/not_empty_measures_alt.nc | Bin 0 -> 45477 bytes tests/datafiles/not_empty_mur.nc4 | Bin 0 -> 60907 bytes tests/datafiles/not_empty_smap.h5 | Bin 0 -> 3000192 bytes tests/datafiles/not_empty_wswm.nc | Bin 0 -> 33119 bytes tests/datafiles/partial_empty_mur.nc4 | Bin 0 -> 84738 bytes .../ascat_longitude_more_than_180.bin | Bin 0 -> 3858 bytes .../ascatb_nonempty_nexustile.bin | Bin 0 -> 3515 bytes .../avhrr_nonempty_nexustile.bin | Bin 0 -> 892 bytes .../ccmp_nonempty_nexustile.bin | Bin 0 -> 27427 bytes .../smap_nonempty_nexustile.bin | Bin 0 -> 1374 bytes tests/hd5splitter.py | 123 +++++++ tests/kelvintocelsius_test.py | 40 +++ tests/processorchain_test.py | 92 ++++++ tests/regrid1x1_test.py | 79 +++++ tests/subtract180longitude_test.py | 57 ++++ tests/tilereadingprocessor_test.py | 330 +++++++++++++++++++ tests/tilesumarizingprocessor_test.py | 80 +++++ tests/winddirspeedtouv_test.py | 89 +++++ 44 files changed, 2275 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba846b9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.pyc + +.idea + +.DS_Store + +*.egg-info +build +dist http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/__init__.py ---------------------------------------------------------------------- diff --git a/ningesterpy/__init__.py b/ningesterpy/__init__.py new file mode 100644 index 0000000..86f81aa --- /dev/null +++ b/ningesterpy/__init__.py @@ -0,0 +1,5 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +from ningesterpy import ningesterpy \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/ningesterpy.py ---------------------------------------------------------------------- diff --git a/ningesterpy/ningesterpy.py b/ningesterpy/ningesterpy.py new file mode 100644 index 0000000..e9c4fde --- /dev/null +++ b/ningesterpy/ningesterpy.py @@ -0,0 +1,80 @@ +import logging +import uuid + +import nexusproto.NexusContent_pb2 as nexusproto +from flask import Flask, request, jsonify, Response +from flask.json import JSONEncoder +from flask_accept import accept +from google.protobuf import json_format +from werkzeug.exceptions import HTTPException, BadRequest +from werkzeug.exceptions import default_exceptions + +from processors.processorchain import ProcessorChain, ProcessorNotFound, MissingProcessorArguments + +applog = logging.getLogger(__name__) +app = Flask(__name__) + + +class ProtobufJSONEncoder(JSONEncoder): + def default(self, obj): + try: + if isinstance(obj, nexusproto.NexusTile): + json_obj = json_format.MessageToJson(obj) + return json_obj + iterable = iter(obj) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, obj) + + [email protected]('/processorchain', methods=['POST'], ) +@accept('application/octet-stream', '*/*') +def run_processor_chain(): + try: + parameters = request.get_json() + except Exception as e: + raise BadRequest("Invalid JSON data") from e + + try: + processor_list = parameters['processor_list'] + except (KeyError, TypeError): + raise BadRequest(description="processor_list is required.") + + try: + chain = ProcessorChain(processor_list) + except ProcessorNotFound as e: + raise BadRequest("Unknown processor requested: %s" % e.missing_processor) from e + except MissingProcessorArguments as e: + raise BadRequest( + "%s missing required configuration options: %s" % (e.processor, e.missing_processor_args)) from e + + input_data = parameters['input_data'] + + result = next(chain.process(input_data), None) + + if isinstance(result, nexusproto.NexusTile): + result = result.SerializeToString() + + return Response(result, mimetype='application/octet-stream') + + +def handle_error(e): + error_id = uuid.uuid4() + + app.logger.exception("Exception %s" % error_id) + code = 500 + message = "Internal server error" + if isinstance(e, HTTPException): + code = e.code + message = str(e) + return jsonify(message=message, error_id=error_id), code + + +if __name__ == '__main__': + app.register_error_handler(Exception, handle_error) + for ex in default_exceptions: + app.register_error_handler(ex, handle_error) + app.json_encoder = ProtobufJSONEncoder + app.run() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/__init__.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/__init__.py b/ningesterpy/processors/__init__.py new file mode 100644 index 0000000..5bb9094 --- /dev/null +++ b/ningesterpy/processors/__init__.py @@ -0,0 +1,66 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +from collections import defaultdict + +import nexusproto.NexusContent_pb2 as nexusproto + + +class Processor(object): + def __init__(self, *args, **kwargs): + self.environ = defaultdict(lambda: None) + for k, v in kwargs.items(): + self.environ[k.upper()] = v + pass + + def process(self, input_data): + raise NotImplementedError + + +class NexusTileProcessor(Processor): + @staticmethod + def parse_input(input_data): + if isinstance(input_data, nexusproto.NexusTile): + return input_data + else: + return nexusproto.NexusTile.FromString(input_data) + + def process(self, input_data): + nexus_tile = self.parse_input(input_data) + + for data in self.process_nexus_tile(nexus_tile): + yield data + + def process_nexus_tile(self, nexus_tile): + raise NotImplementedError + +# All installed processors need to be imported and added to the dict below + +from processors.callncpdq import CallNcpdq +from processors.callncra import CallNcra +from processors.computespeeddirfromuv import ComputeSpeedDirFromUV +from processors.emptytilefilter import EmptyTileFilter +from processors.kelvintocelsius import KelvinToCelsius +from processors.normalizetimebeginningofmonth import NormalizeTimeBeginningOfMonth +from processors.regrid1x1 import Regrid1x1 +from processors.subtract180longitude import Subtract180Longitude +from processors.tilereadingprocessor import GridReadingProcessor, SwathReadingProcessor, TimeSeriesReadingProcessor +from processors.tilesummarizingprocessor import TileSummarizingProcessor +from processors.winddirspeedtouv import WindDirSpeedToUV + +INSTALLED_PROCESSORS = { + "CallNcpdq": CallNcpdq, + "CallNcra": CallNcra, + "ComputeSpeedDirFromUV": ComputeSpeedDirFromUV, + "EmptyTileFilter": EmptyTileFilter, + "KelvinToCelsius": KelvinToCelsius, + "NormalizeTimeBeginningOfMonth": NormalizeTimeBeginningOfMonth, + "Regrid1x1": Regrid1x1, + "Subtract180Longitude": Subtract180Longitude, + "GridReadingProcessor": GridReadingProcessor, + "SwathReadingProcessor": SwathReadingProcessor, + "TimeSeriesReadingProcessor": TimeSeriesReadingProcessor, + "TileSummarizingProcessor": TileSummarizingProcessor, + "WindDirSpeedToUV": WindDirSpeedToUV +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/callncpdq.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/callncpdq.py b/ningesterpy/processors/callncpdq.py new file mode 100644 index 0000000..3fee959 --- /dev/null +++ b/ningesterpy/processors/callncpdq.py @@ -0,0 +1,48 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + +import logging +import os +from subprocess import call + +from processors import Processor + + +class CallNcpdq(Processor): + + def __init__(self, dimension_order, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.dimension_order = dimension_order + self.output_prefix = self.environ.get("OUTPUT_PREFIX", 'permuted_') + self.permute_variable = self.environ["PERMUTE_VARIABLE"] + + def process(self, input_data): + """ + input_data: Path to input netCDF file + + If environment variable `PERMUTE_VARIABLE` is not set: + Calls ``ncpdq -a ${DIMENSION_ORDER} in_path ${OUTPUT_PREFIX}in_path`` + Otherwise: + Calls ``ncpdq -v ${PERMUTE_VARIABLE} -a ${DIMENSION_ORDER} in_path ${OUTPUT_PREFIX}in_path`` + """ + + output_filename = self.output_prefix + os.path.basename(input_data) + output_path = os.path.join(os.path.dirname(input_data), output_filename) + + command = ['ncpdq', '-a', ','.join(self.dimension_order)] + + if self.permute_variable: + command.append('-v') + command.append(self.permute_variable) + + command.append(input_data) + command.append(output_path) + + logging.debug('Calling command %s' % ' '.join(command)) + retcode = call(command) + logging.debug('Command returned exit code %d' % retcode) + + yield output_path http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/callncra.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/callncra.py b/ningesterpy/processors/callncra.py new file mode 100644 index 0000000..d00e75b --- /dev/null +++ b/ningesterpy/processors/callncra.py @@ -0,0 +1,48 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + +import glob +import os +from subprocess import call + +from netCDF4 import Dataset, num2date + +from processors import Processor + + +class CallNcra(Processor): + def __init__(self, output_filename_pattern, time_var_name, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.output_filename_pattern = output_filename_pattern + self.time_var_name = time_var_name + + self.glob_pattern = self.environ.get("FILEMATCH_PATTERN", '*.nc') + + def process(self, in_path): + target_datetime = self.get_datetime_from_dataset(in_path) + target_yearmonth = target_datetime.strftime('%Y%m') + + output_filename = target_datetime.strftime(self.output_filename_pattern) + output_path = os.path.join(os.path.dirname(in_path), output_filename) + + datasets = glob.glob(os.path.join(os.path.dirname(in_path), self.glob_pattern)) + + datasets_to_average = [dataset_path for dataset_path in datasets if + self.get_datetime_from_dataset(dataset_path).strftime('%Y%m') == target_yearmonth] + + command = ['ncra', '-O'] + command.extend(datasets_to_average) + command.append(output_path) + call(command) + + yield output_path + + def get_datetime_from_dataset(self, dataset_path): + with Dataset(dataset_path) as dataset_in: + time_units = getattr(dataset_in[self.time_var_name], 'units', None) + calendar = getattr(dataset_in[self.time_var_name], 'calendar', 'standard') + thedatetime = num2date(dataset_in[self.time_var_name][:].item(), units=time_units, calendar=calendar) + return thedatetime http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/computespeeddirfromuv.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/computespeeddirfromuv.py b/ningesterpy/processors/computespeeddirfromuv.py new file mode 100644 index 0000000..284a668 --- /dev/null +++ b/ningesterpy/processors/computespeeddirfromuv.py @@ -0,0 +1,68 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + +import numpy +from nexusproto.serialization import from_shaped_array, to_shaped_array + +from processors import NexusTileProcessor + + +def calculate_speed_direction(wind_u, wind_v): + speed = numpy.sqrt(numpy.add(numpy.multiply(wind_u, wind_u), numpy.multiply(wind_v, wind_v))) + direction = numpy.degrees(numpy.arctan2(-wind_u, -wind_v)) % 360 + return speed, direction + + +class ComputeSpeedDirFromUV(NexusTileProcessor): + + def __init__(self, wind_u_var_name, wind_v_var_name, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.wind_u_var_name = wind_u_var_name + self.wind_v_var_name = wind_v_var_name + + def process_nexus_tile(self, nexus_tile): + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + # Either wind_u or wind_v are in meta. Whichever is not in meta is in variable_data + try: + wind_v = next(meta for meta in the_tile_data.meta_data if meta.name == self.wind_v_var_name).meta_data + wind_u = the_tile_data.variable_data + except StopIteration: + try: + wind_u = next(meta for meta in the_tile_data.meta_data if meta.name == self.wind_u_var_name).meta_data + wind_v = the_tile_data.variable_data + except StopIteration: + if hasattr(nexus_tile, "summary"): + raise RuntimeError( + "Neither wind_u nor wind_v were found in the meta data for granule %s slice %s." + " Cannot compute wind speed or direction." % ( + getattr(nexus_tile.summary, "granule", "unknown"), + getattr(nexus_tile.summary, "section_spec", "unknown"))) + else: + raise RuntimeError( + "Neither wind_u nor wind_v were found in the meta data. Cannot compute wind speed or direction.") + + wind_u = from_shaped_array(wind_u) + wind_v = from_shaped_array(wind_v) + + assert wind_u.shape == wind_v.shape + + # Do calculation + wind_speed_data, wind_dir_data = calculate_speed_direction(wind_u, wind_v) + + # Add wind_speed to meta data + wind_speed_meta = the_tile_data.meta_data.add() + wind_speed_meta.name = 'wind_speed' + wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed_data)) + + # Add wind_dir to meta data + wind_dir_meta = the_tile_data.meta_data.add() + wind_dir_meta.name = 'wind_dir' + wind_dir_meta.meta_data.CopyFrom(to_shaped_array(wind_dir_data)) + + yield nexus_tile http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/emptytilefilter.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/emptytilefilter.py b/ningesterpy/processors/emptytilefilter.py new file mode 100644 index 0000000..ec30d2e --- /dev/null +++ b/ningesterpy/processors/emptytilefilter.py @@ -0,0 +1,31 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import nexusproto.NexusContent_pb2 as nexusproto +import numpy +import logging +from nexusproto.serialization import from_shaped_array + +from processors import NexusTileProcessor + +logger = logging.getLogger('emptytilefilter') + +def parse_input(nexus_tile_data): + return nexusproto.NexusTile.FromString(nexus_tile_data) + + +class EmptyTileFilter(NexusTileProcessor): + def process_nexus_tile(self, nexus_tile): + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + data = from_shaped_array(the_tile_data.variable_data) + + # Only supply data if there is actual values in the tile + if data.size - numpy.count_nonzero(numpy.isnan(data)) > 0: + yield nexus_tile + elif nexus_tile.HasField("summary"): + logger.warning("Discarding data %s from %s because it is empty" % ( + nexus_tile.summary.section_spec, nexus_tile.summary.granule)) http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/kelvintocelsius.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/kelvintocelsius.py b/ningesterpy/processors/kelvintocelsius.py new file mode 100644 index 0000000..361a50d --- /dev/null +++ b/ningesterpy/processors/kelvintocelsius.py @@ -0,0 +1,21 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + +from nexusproto.serialization import from_shaped_array, to_shaped_array + +from processors import NexusTileProcessor + + +class KelvinToCelsius(NexusTileProcessor): + def process_nexus_tile(self, nexus_tile): + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + var_data = from_shaped_array(the_tile_data.variable_data) - 273.15 + + the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data)) + + yield nexus_tile http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/normalizetimebeginningofmonth.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/normalizetimebeginningofmonth.py b/ningesterpy/processors/normalizetimebeginningofmonth.py new file mode 100644 index 0000000..2be593c --- /dev/null +++ b/ningesterpy/processors/normalizetimebeginningofmonth.py @@ -0,0 +1,30 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import datetime + +from pytz import timezone + +from processors import NexusTileProcessor + +EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1)) + + +class NormalizeTimeBeginningOfMonth(NexusTileProcessor): + def process_nexus_tile(self, nexus_tile): + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + time = the_tile_data.time + + timeObj = datetime.datetime.utcfromtimestamp(time) + + timeObj = timeObj.replace(day=1) + + timeObj = timezone('UTC').localize(timeObj) + + the_tile_data.time = int((timeObj - EPOCH).total_seconds()) + + yield nexus_tile http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/processorchain.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/processorchain.py b/ningesterpy/processors/processorchain.py new file mode 100644 index 0000000..aea605e --- /dev/null +++ b/ningesterpy/processors/processorchain.py @@ -0,0 +1,72 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import inspect + +import processors + + +class BadChainException(Exception): + pass + + +class ProcessorNotFound(Exception): + def __init__(self, missing_processor, *args): + message = "Processor %s is not defined in INSTALLED_PROCESSORS. See processors/__init__.py" % missing_processor + + self.missing_processor = missing_processor + super().__init__(message, *args) + + +class MissingProcessorArguments(Exception): + def __init__(self, processor, missing_processor_args, *args): + message = "%s is missing required arguments: %s" % (processor, missing_processor_args) + + self.processor = processor + self.missing_processor_args = missing_processor_args + super().__init__(message, *args) + + +class ProcessorChain(processors.Processor): + def __init__(self, processor_list, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.processors = [] + # Attempt to construct the needed processors + for processor in processor_list: + try: + processor_constructor = processors.INSTALLED_PROCESSORS[processor['name']] + except KeyError as e: + raise ProcessorNotFound(processor['name']) from e + + missing_args = [] + for arg in inspect.signature(processor_constructor).parameters.keys(): + if arg in ['args', 'kwargs']: + continue + if arg not in processor['config']: + missing_args.append(arg) + + if missing_args: + raise MissingProcessorArguments(processor['name'], missing_args) + + if 'config' in processor.keys(): + processor_instance = processor_constructor(**processor['config']) + else: + processor_instance = processor_constructor() + + self.processors.append(processor_instance) + + def process(self, input_data): + + def recursive_processing_chain(gen_index, message): + + next_gen = self.processors[gen_index + 1].process(message) + for next_message in next_gen: + if gen_index + 1 == len(self.processors) - 1: + yield next_message + else: + for result in recursive_processing_chain(gen_index + 1, next_message): + yield result + + return recursive_processing_chain(-1, input_data) http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/regrid1x1.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/regrid1x1.py b/ningesterpy/processors/regrid1x1.py new file mode 100644 index 0000000..0248f1e --- /dev/null +++ b/ningesterpy/processors/regrid1x1.py @@ -0,0 +1,136 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + +import os +from datetime import datetime + +import numpy as np +from netCDF4 import Dataset +from pytz import timezone +from scipy import interpolate + +from processors import Processor + +UTC = timezone('UTC') +ISO_8601 = '%Y-%m-%dT%H:%M:%S%z' + + +class Regrid1x1(Processor): + + def __init__(self, variables_to_regrid, latitude_var_name, longitude_var_name, time_var_name, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.variables_to_regrid = variables_to_regrid + self.latitude_var_name = latitude_var_name + self.longitude_var_name = longitude_var_name + self.time_var_name = time_var_name + + self.filename_prefix = self.environ.get("FILENAME_PREFIX", '1x1regrid-') + + vvr = self.environ['VARIABLE_VALID_RANGE'] + if vvr: + vvr_iter = iter(vvr.split(':')) + self.variable_valid_range = {varrange[0]: [varrange[1], varrange[2]] for varrange in + zip(vvr_iter, vvr_iter, vvr_iter)} + else: + self.variable_valid_range = {} + + def process(self, in_filepath): + in_path = os.path.join('/', *in_filepath.split(os.sep)[0:-1]) + out_filepath = os.path.join(in_path, self.filename_prefix + in_filepath.split(os.sep)[-1]) + + with Dataset(in_filepath) as inputds: + in_lon = inputds[self.longitude_var_name] + in_lat = inputds[self.latitude_var_name] + in_time = inputds[self.time_var_name] + + lon1deg = np.arange(np.floor(np.min(in_lon)), np.ceil(np.max(in_lon)), 1) + lat1deg = np.arange(np.floor(np.min(in_lat)), np.ceil(np.max(in_lat)), 1) + out_time = np.array(in_time) + + with Dataset(out_filepath, mode='w') as outputds: + outputds.createDimension(self.longitude_var_name, len(lon1deg)) + outputds.createVariable(self.longitude_var_name, in_lon.dtype, dimensions=(self.longitude_var_name,)) + outputds[self.longitude_var_name][:] = lon1deg + outputds[self.longitude_var_name].setncatts( + {attrname: inputds[self.longitude_var_name].getncattr(attrname) for attrname in + inputds[self.longitude_var_name].ncattrs() if + str(attrname) not in ['bounds', 'valid_min', 'valid_max']}) + + outputds.createDimension(self.latitude_var_name, len(lat1deg)) + outputds.createVariable(self.latitude_var_name, in_lat.dtype, dimensions=(self.latitude_var_name,)) + outputds[self.latitude_var_name][:] = lat1deg + outputds[self.latitude_var_name].setncatts( + {attrname: inputds[self.latitude_var_name].getncattr(attrname) for attrname in + inputds[self.latitude_var_name].ncattrs() if + str(attrname) not in ['bounds', 'valid_min', 'valid_max']}) + + outputds.createDimension(self.time_var_name) + outputds.createVariable(self.time_var_name, inputds[self.time_var_name].dtype, + dimensions=(self.time_var_name,)) + outputds[self.time_var_name][:] = out_time + outputds[self.time_var_name].setncatts( + {attrname: inputds[self.time_var_name].getncattr(attrname) for attrname in + inputds[self.time_var_name].ncattrs() + if + str(attrname) != 'bounds'}) + + for variable_name in self.variables_to_regrid.split(','): + + # If longitude is the first dimension, we need to transpose the dimensions + transpose_dimensions = inputds[variable_name].dimensions == ( + self.time_var_name, self.longitude_var_name, self.latitude_var_name) + + outputds.createVariable(variable_name, inputds[variable_name].dtype, + dimensions=inputds[variable_name].dimensions) + outputds[variable_name].setncatts( + {attrname: inputds[variable_name].getncattr(attrname) for attrname in + inputds[variable_name].ncattrs()}) + if variable_name in self.variable_valid_range.keys(): + outputds[variable_name].valid_range = [ + np.array([self.variable_valid_range[variable_name][0]], + dtype=inputds[variable_name].dtype).item(), + np.array([self.variable_valid_range[variable_name][1]], + dtype=inputds[variable_name].dtype).item()] + + for ti in range(0, len(out_time)): + in_data = inputds[variable_name][ti, :, :] + if transpose_dimensions: + in_data = in_data.T + + # Produces erroneous values on the edges of data + # interp_func = interpolate.interp2d(in_lon[:], in_lat[:], in_data[:], fill_value=float('NaN')) + + x_mesh, y_mesh = np.meshgrid(in_lon[:], in_lat[:], copy=False) + + # Does not work for large datasets (n > 5000) + # interp_func = interpolate.Rbf(x_mesh, y_mesh, in_data[:], function='linear', smooth=0) + + x1_mesh, y1_mesh = np.meshgrid(lon1deg, lat1deg, copy=False) + out_data = interpolate.griddata(np.array([x_mesh.ravel(), y_mesh.ravel()]).T, in_data.ravel(), + (x1_mesh, y1_mesh), method='nearest') + + if transpose_dimensions: + out_data = out_data.T + + outputds[variable_name][ti, :] = out_data[np.newaxis, :] + + global_atts = { + 'geospatial_lon_min': np.float(np.min(lon1deg)), + 'geospatial_lon_max': np.float(np.max(lon1deg)), + 'geospatial_lat_min': np.float(np.min(lat1deg)), + 'geospatial_lat_max': np.float(np.max(lat1deg)), + 'Conventions': 'CF-1.6', + 'date_created': datetime.utcnow().replace(tzinfo=UTC).strftime(ISO_8601), + 'title': getattr(inputds, 'title', ''), + 'time_coverage_start': getattr(inputds, 'time_coverage_start', ''), + 'time_coverage_end': getattr(inputds, 'time_coverage_end', ''), + 'Institution': getattr(inputds, 'Institution', ''), + 'summary': getattr(inputds, 'summary', ''), + } + + outputds.setncatts(global_atts) + + yield out_filepath http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/subtract180longitude.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/subtract180longitude.py b/ningesterpy/processors/subtract180longitude.py new file mode 100644 index 0000000..b6ed693 --- /dev/null +++ b/ningesterpy/processors/subtract180longitude.py @@ -0,0 +1,32 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +from nexusproto.serialization import from_shaped_array, to_shaped_array + +from processors import NexusTileProcessor + + +class Subtract180Longitude(NexusTileProcessor): + def process_nexus_tile(self, nexus_tile): + """ + This method will transform longitude values in degrees_east from 0 TO 360 to -180 to 180 + + :param self: + :param nexus_tile: The nexus_tile + :return: Tile data with altered longitude values + """ + + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + longitudes = from_shaped_array(the_tile_data.longitude) + + # Only subtract 360 if the longitude is greater than 180 + longitudes[longitudes > 180] -= 360 + + the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes)) + + yield nexus_tile + http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/tilereadingprocessor.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/tilereadingprocessor.py b/ningesterpy/processors/tilereadingprocessor.py new file mode 100644 index 0000000..de63634 --- /dev/null +++ b/ningesterpy/processors/tilereadingprocessor.py @@ -0,0 +1,266 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import datetime +from collections import OrderedDict +from contextlib import contextmanager +from os import sep, path, remove +from urllib.request import urlopen + +import nexusproto.NexusContent_pb2 as nexusproto +import numpy +from netCDF4 import Dataset, num2date +from nexusproto.serialization import to_shaped_array, to_metadata +from pytz import timezone + +from processors import Processor + +EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1)) + + +@contextmanager +def closing(thing): + try: + yield thing + finally: + thing.close() + + +def parse_input(the_input, temp_dir): + # Split string on ';' + specs_and_path = [str(part).strip() for part in str(the_input).split(';')] + + # Tile specifications are all but the last element + specs = specs_and_path[:-1] + # Generate a list of tuples, where each tuple is a (string, map) that represents a + # tile spec in the form (str(section_spec), { dimension_name : slice, dimension2_name : slice }) + tile_specifications = [slices_from_spec(section_spec) for section_spec in specs] + + # The path is the last element of the input split by ';' + file_path = specs_and_path[-1] + file_name = file_path.split(sep)[-1] + # If given a temporary directory location, copy the file to the temporary directory and return that path + if temp_dir is not None: + temp_file_path = path.join(temp_dir, file_name) + with closing(urlopen(file_path)) as original_granule: + with open(temp_file_path, 'wb') as temp_granule: + for chunk in iter((lambda: original_granule.read(512000)), ''): + temp_granule.write(chunk) + + file_path = temp_file_path + + # Remove file:// if it's there because netcdf lib doesn't like it + file_path = file_path[len('file://'):] if file_path.startswith('file://') else file_path + + return tile_specifications, file_path + + +def slices_from_spec(spec): + dimtoslice = {} + for dimension in spec.split(','): + name, start, stop = dimension.split(':') + dimtoslice[name] = slice(int(start), int(stop)) + + return spec, dimtoslice + + +def to_seconds_from_epoch(date, timeunits=None, start_day=None, timeoffset=None): + try: + date = num2date(date, units=timeunits) + except ValueError: + assert isinstance(start_day, datetime.date), "start_day is not a datetime.date object" + the_datetime = datetime.datetime.combine(start_day, datetime.datetime.min.time()) + date = the_datetime + datetime.timedelta(seconds=date) + + if isinstance(date, datetime.datetime): + date = timezone('UTC').localize(date) + else: + date = timezone('UTC').localize(datetime.datetime.strptime(str(date), '%Y-%m-%d %H:%M:%S')) + + if timeoffset is not None: + return int((date - EPOCH).total_seconds()) + timeoffset + else: + return int((date - EPOCH).total_seconds()) + + +def get_ordered_slices(ds, variable, dimension_to_slice): + dimensions_for_variable = [str(dimension) for dimension in ds[variable].dimensions] + ordered_slices = OrderedDict() + for dimension in dimensions_for_variable: + ordered_slices[dimension] = dimension_to_slice[dimension] + return ordered_slices + + +def new_nexus_tile(file_path, section_spec): + nexus_tile = nexusproto.NexusTile() + tile_summary = nexusproto.TileSummary() + tile_summary.granule = file_path.split(sep)[-1] + tile_summary.section_spec = section_spec + nexus_tile.summary.CopyFrom(tile_summary) + return nexus_tile + + +class TileReadingProcessor(Processor): + def __init__(self, variable_to_read, latitude, longitude, *args, **kwargs): + super().__init__(*args, **kwargs) + # Required properties for all reader types + self.variable_to_read = variable_to_read + self.latitude = latitude + self.longitude = longitude + + # Common optional properties + self.temp_dir = self.environ['TEMP_DIR'] + self.metadata = self.environ['META'] + self.start_of_day = self.environ['GLBLATTR_DAY'] + self.start_of_day_pattern = self.environ['GLBLATTR_DAY_FORMAT'] + self.time_offset = int(self.environ['TIME_OFFSET']) if self.environ['TIME_OFFSET'] is not None else None + + def process(self, input_data): + tile_specifications, file_path = parse_input(input_data, self.temp_dir) + + for data in self.read_data(tile_specifications, file_path): + yield data + + # If temp dir is defined, delete the temporary file + if self.temp_dir is not None: + remove(file_path) + + def read_data(self, tile_specifications, file_path): + raise NotImplementedError + + +class GridReadingProcessor(TileReadingProcessor): + def read_data(self, tile_specifications, file_path): + # Time is optional for Grid data + time = self.environ['TIME'] + + with Dataset(file_path) as ds: + for section_spec, dimtoslice in tile_specifications: + tile = nexusproto.GridTile() + + tile.latitude.CopyFrom( + to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[self.latitude]], numpy.NaN))) + + tile.longitude.CopyFrom( + to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[self.longitude]], numpy.NaN))) + + # Before we read the data we need to make sure the dimensions are in the proper order so we don't have any + # indexing issues + ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice) + # Read data using the ordered slices, replacing masked values with NaN + data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN) + + tile.variable_data.CopyFrom(to_shaped_array(data_array)) + + if self.metadata is not None: + tile.meta_data.add().CopyFrom( + to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())])) + + if time is not None: + timevar = ds[time] + # Note assumption is that index of time is start value in dimtoslice + tile.time = to_seconds_from_epoch(timevar[dimtoslice[time].start], + timeunits=timevar.getncattr('units'), + timeoffset=self.time_offset) + + nexus_tile = new_nexus_tile(file_path, section_spec) + nexus_tile.tile.grid_tile.CopyFrom(tile) + + yield nexus_tile + + +class SwathReadingProcessor(TileReadingProcessor): + def __init__(self, variable_to_read, latitude, longitude, time, **kwargs): + super().__init__(variable_to_read, latitude, longitude, **kwargs) + + # Time is required for swath data + self.time = time + + def read_data(self, tile_specifications, file_path): + with Dataset(file_path) as ds: + for section_spec, dimtoslice in tile_specifications: + tile = nexusproto.SwathTile() + # Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable + ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice) + tile.latitude.CopyFrom( + to_shaped_array(numpy.ma.filled(ds[self.latitude][tuple(ordered_slices.values())], numpy.NaN))) + + tile.longitude.CopyFrom( + to_shaped_array(numpy.ma.filled(ds[self.longitude][tuple(ordered_slices.values())], numpy.NaN))) + + timetile = ds[self.time][ + tuple([ordered_slices[time_dim] for time_dim in ds[self.time].dimensions])].astype( + 'float64', + casting='same_kind', + copy=False) + timeunits = ds[self.time].getncattr('units') + try: + start_of_day_date = datetime.datetime.strptime(ds.getncattr(self.start_of_day), + self.start_of_day_pattern) + except Exception: + start_of_day_date = None + + for index in numpy.ndindex(timetile.shape): + timetile[index] = to_seconds_from_epoch(timetile[index].item(), timeunits=timeunits, + start_day=start_of_day_date, timeoffset=self.time_offset) + + tile.time.CopyFrom(to_shaped_array(timetile)) + + # Read the data converting masked values to NaN + data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN) + tile.variable_data.CopyFrom(to_shaped_array(data_array)) + + if self.metadata is not None: + tile.meta_data.add().CopyFrom( + to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())])) + + nexus_tile = new_nexus_tile(file_path, section_spec) + nexus_tile.tile.swath_tile.CopyFrom(tile) + + yield nexus_tile + + +class TimeSeriesReadingProcessor(TileReadingProcessor): + def __init__(self, variable_to_read, latitude, longitude, time, **kwargs): + super().__init__(variable_to_read, latitude, longitude, **kwargs) + + # Time is required for swath data + self.time = time + + def read_data(self, tile_specifications, file_path): + with Dataset(file_path) as ds: + for section_spec, dimtoslice in tile_specifications: + tile = nexusproto.TimeSeriesTile() + + instance_dimension = next( + iter([dim for dim in ds[self.variable_to_read].dimensions if dim != self.time])) + + tile.latitude.CopyFrom( + to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[instance_dimension]], numpy.NaN))) + + tile.longitude.CopyFrom( + to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[instance_dimension]], numpy.NaN))) + + # Before we read the data we need to make sure the dimensions are in the proper order so we don't + # have any indexing issues + ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice) + # Read data using the ordered slices, replacing masked values with NaN + data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN) + + tile.variable_data.CopyFrom(to_shaped_array(data_array)) + + if self.metadata is not None: + tile.meta_data.add().CopyFrom( + to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())])) + + timevar = ds[self.time] + # Note assumption is that index of time is start value in dimtoslice + tile.time = to_seconds_from_epoch(timevar[dimtoslice[self.time].start], + timeunits=timevar.getncattr('units'), + timeoffset=self.time_offset) + + nexus_tile = new_nexus_tile(file_path, section_spec) + nexus_tile.tile.time_series_tile.CopyFrom(tile) + + yield nexus_tile http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/tilesummarizingprocessor.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/tilesummarizingprocessor.py b/ningesterpy/processors/tilesummarizingprocessor.py new file mode 100644 index 0000000..970a251 --- /dev/null +++ b/ningesterpy/processors/tilesummarizingprocessor.py @@ -0,0 +1,93 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import nexusproto.NexusContent_pb2 as nexusproto +import numpy +from nexusproto.serialization import from_shaped_array + +from processors import NexusTileProcessor + + +class NoTimeException(Exception): + pass + + +def find_time_min_max(tile_data): + # Only try to grab min/max time if it exists as a ShapedArray + if tile_data.HasField("time") and isinstance(tile_data.time, nexusproto.ShapedArray): + time_data = from_shaped_array(tile_data.time) + min_time = int(numpy.nanmin(time_data).item()) + max_time = int(numpy.nanmax(time_data).item()) + + return min_time, max_time + elif tile_data.HasField("time") and isinstance(tile_data.time, int): + return tile_data.time, tile_data.time + + raise NoTimeException + + +class TileSummarizingProcessor(NexusTileProcessor): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.stored_var_name = self.environ['STORED_VAR_NAME'] + + def process_nexus_tile(self, nexus_tile): + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + latitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.latitude)) + longitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.longitude)) + + data = from_shaped_array(the_tile_data.variable_data) + + if nexus_tile.HasField("summary"): + tilesummary = nexus_tile.summary + else: + tilesummary = nexusproto.TileSummary() + + tilesummary.bbox.lat_min = numpy.nanmin(latitudes).item() + tilesummary.bbox.lat_max = numpy.nanmax(latitudes).item() + tilesummary.bbox.lon_min = numpy.nanmin(longitudes).item() + tilesummary.bbox.lon_max = numpy.nanmax(longitudes).item() + + tilesummary.stats.min = numpy.nanmin(data).item() + tilesummary.stats.max = numpy.nanmax(data).item() + + # In order to accurately calculate the average we need to weight the data based on the cosine of its latitude + # This is handled slightly differently for swath vs. grid data + if the_tile_type == 'swath_tile': + # For Swath tiles, len(data) == len(latitudes) == len(longitudes). So we can simply weight each element in the + # data array + tilesummary.stats.mean = numpy.ma.average(numpy.ma.masked_invalid(data), + weights=numpy.cos(numpy.radians(latitudes))).item() + elif the_tile_type == 'grid_tile': + # Grid tiles need to repeat the weight for every longitude + # TODO This assumes data axis' are ordered as latitude x longitude + tilesummary.stats.mean = numpy.ma.average(numpy.ma.masked_invalid(data).flatten(), + weights=numpy.cos( + numpy.radians( + numpy.repeat(latitudes, len(longitudes))))).item() + else: + # Default to simple average with no weighting + tilesummary.stats.mean = numpy.nanmean(data).item() + + tilesummary.stats.count = data.size - numpy.count_nonzero(numpy.isnan(data)) + + try: + min_time, max_time = find_time_min_max(the_tile_data) + tilesummary.stats.min_time = min_time + tilesummary.stats.max_time = max_time + except NoTimeException: + pass + + try: + tilesummary.data_var_name = self.stored_var_name + except TypeError: + pass + + nexus_tile.summary.CopyFrom(tilesummary) + yield nexus_tile http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/ningesterpy/processors/winddirspeedtouv.py ---------------------------------------------------------------------- diff --git a/ningesterpy/processors/winddirspeedtouv.py b/ningesterpy/processors/winddirspeedtouv.py new file mode 100644 index 0000000..9a4445d --- /dev/null +++ b/ningesterpy/processors/winddirspeedtouv.py @@ -0,0 +1,90 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +from math import cos +from math import radians +from math import sin + +import numpy +from nexusproto.serialization import from_shaped_array, to_shaped_array + +from processors import NexusTileProcessor + + +def enum(**enums): + return type('Enum', (), enums) + + +U_OR_V_ENUM = enum(U='u', V='v') + + +def calculate_u_component_value(direction, speed): + if direction is numpy.ma.masked or speed is numpy.ma.masked: + return numpy.ma.masked + + return speed * sin(direction) + + +def calculate_v_component_value(direction, speed): + if direction is numpy.ma.masked or speed is numpy.ma.masked: + return numpy.ma.masked + + return speed * cos(direction) + + +class WindDirSpeedToUV(NexusTileProcessor): + + def __init__(self, u_or_v, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.u_or_v = u_or_v.lower() + + def process_nexus_tile(self, nexus_tile): + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + wind_speed = from_shaped_array(the_tile_data.variable_data) + + wind_dir = from_shaped_array( + next(meta for meta in the_tile_data.meta_data if meta.name == 'wind_dir').meta_data) + + assert wind_speed.shape == wind_dir.shape + + wind_u_component = numpy.ma.empty(wind_speed.shape, dtype=float) + wind_v_component = numpy.ma.empty(wind_speed.shape, dtype=float) + wind_speed_iter = numpy.nditer(wind_speed, flags=['multi_index']) + while not wind_speed_iter.finished: + speed = wind_speed_iter[0] + current_index = wind_speed_iter.multi_index + direction = wind_dir[current_index] + + # Convert degrees to radians + direction = radians(direction) + + # Calculate component values + wind_u_component[current_index] = calculate_u_component_value(direction, speed) + wind_v_component[current_index] = calculate_v_component_value(direction, speed) + + wind_speed_iter.iternext() + + # Stick the original data into the meta data + wind_speed_meta = the_tile_data.meta_data.add() + wind_speed_meta.name = 'wind_speed' + wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed)) + + # The u_or_v variable specifies which component variable is the 'data variable' for this tile + # Replace data with the appropriate component value and put the other component in metadata + if self.u_or_v == U_OR_V_ENUM.U: + the_tile_data.variable_data.CopyFrom(to_shaped_array(wind_u_component)) + wind_component_meta = the_tile_data.meta_data.add() + wind_component_meta.name = 'wind_v' + wind_component_meta.meta_data.CopyFrom(to_shaped_array(wind_v_component)) + elif self.u_or_v == U_OR_V_ENUM.V: + the_tile_data.variable_data.CopyFrom(to_shaped_array(wind_v_component)) + wind_component_meta = the_tile_data.meta_data.add() + wind_component_meta.name = 'wind_u' + wind_component_meta.meta_data.CopyFrom(to_shaped_array(wind_u_component)) + + yield nexus_tile http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/requirements.txt ---------------------------------------------------------------------- diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..22d2912 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +werkzeug=0.12.2 +flask=0.12.2 +flask-accept==0.0.4 +nco==4.7.1 +netCDF4==1.3.1 +nexusproto==0.41 +numpy==1.12.1 +protobuf==3.2.0 +pytz==2017.2 +PyYAML==3.12 +scipy==0.18.1 +six==1.10.0 http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f53f9ea --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +from setuptools import setup, find_packages + +__version__ = '0.1' + +setup( + name="ningesterpy", + version=__version__, + url="https://github.jpl.nasa.gov/thuang/nexus";, + + author="Team Nexus", + + description="Python modules that can be used for NEXUS ingest.", + # long_description=open('README.md').read(), + + packages=find_packages(), + test_suite="tests", + platforms='any', + + classifiers=[ + 'Development Status :: 1 - Pre-Alpha', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3.5', + ] +) http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/__init__.py ---------------------------------------------------------------------- diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..bd9282c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/callncpdq_test.py ---------------------------------------------------------------------- diff --git a/tests/callncpdq_test.py b/tests/callncpdq_test.py new file mode 100644 index 0000000..388a70b --- /dev/null +++ b/tests/callncpdq_test.py @@ -0,0 +1,53 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import subprocess +import unittest +from os import path, remove + +from netCDF4 import Dataset + +from processors import callncpdq + + +class TestMeasuresData(unittest.TestCase): + + @unittest.skipIf(int(subprocess.call(["ncpdq", "-r"])) not in {0, 1}, "requires ncpdq") + def test_permute_all_variables(self): + dimension_order = ['Time', 'Latitude', 'Longitude'] + + the_module = callncpdq.CallNcpdq(dimension_order) + + expected_dimensions = dimension_order + + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc') + + output_path = list(the_module.process(test_file))[0] + + with Dataset(output_path) as ds: + sla_var = ds['SLA'] + actual_dimensions = [str(dim) for dim in sla_var.dimensions] + + remove(output_path) + self.assertEqual(expected_dimensions, actual_dimensions) + + @unittest.skipIf(int(subprocess.call(["ncpdq", "-r"])) not in {0, 1}, "requires ncpdq") + def test_permute_one_variable(self): + dimension_order = ['Time', 'Latitude', 'Longitude'] + permute_var = 'SLA' + + the_module = callncpdq.CallNcpdq(dimension_order, environ={"PERMUTE_VARIABLE": permute_var}) + + expected_dimensions = dimension_order + + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc') + + output_path = list(the_module.process(test_file))[0] + + with Dataset(output_path) as ds: + sla_var = ds[permute_var] + actual_dimensions = [str(dim) for dim in sla_var.dimensions] + + remove(output_path) + self.assertEqual(expected_dimensions, actual_dimensions) http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/computespeeddirfromuv_test.py ---------------------------------------------------------------------- diff --git a/tests/computespeeddirfromuv_test.py b/tests/computespeeddirfromuv_test.py new file mode 100644 index 0000000..d880008 --- /dev/null +++ b/tests/computespeeddirfromuv_test.py @@ -0,0 +1,114 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import unittest +from os import path + +import numpy as np +from nexusproto.serialization import from_shaped_array + +import processors +from processors.computespeeddirfromuv import calculate_speed_direction + + +class TestConversion(unittest.TestCase): + def test_dir_from_north(self): + # Negative v means wind is wind blowing to the South + u = 0 + v = -1 + + speed, direction = calculate_speed_direction(u, v) + + # Degrees are where the wind is blowing from (relative to true North) + self.assertEqual(1, speed) + # Wind from North (0 degrees) + self.assertEqual(0, direction) + + def test_dir_from_east(self): + # Negative u means wind is blowing to the West + u = -1 + v = 0 + + speed, direction = calculate_speed_direction(u, v) + + # Degrees are where the wind is blowing from (relative to true North) + self.assertEqual(1, speed) + # Wind from East (90 degrees) + self.assertEqual(90, direction) + + def test_dir_from_south(self): + # Positive v means wind is blowing to the North + u = 0 + v = 1 + + speed, direction = calculate_speed_direction(u, v) + + # Degrees are where the wind is blowing from (relative to true North) + self.assertEqual(1, speed) + # Wind from South (180 degrees) + self.assertEqual(180, direction) + + def test_dir_from_west(self): + # Positive u means wind is blowing to the East + u = 1 + v = 0 + + speed, direction = calculate_speed_direction(u, v) + + # Degrees are where the wind is blowing from (relative to true North) + self.assertEqual(1, speed) + # Wind from West (270 degrees) + self.assertEqual(270, direction) + + def test_speed(self): + # Speed is simply sqrt(u^2 + v^2) + u = 2 + v = 2 + + speed, direction = calculate_speed_direction(u, v) + + self.assertAlmostEqual(2.8284271, speed) + # Wind should be from the southwest + self.assertTrue(180 < direction < 270) + + +class TestCcmpData(unittest.TestCase): + def setUp(self): + self.module = processors.ComputeSpeedDirFromUV('uwnd', 'vwnd') + + def test_speed_dir_computation(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ccmp_nonempty_nexustile.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + results = list(self.module.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile = results[0] + + self.assertTrue(nexus_tile.HasField('tile')) + self.assertTrue(nexus_tile.tile.HasField('grid_tile')) + + # Check data + tile_data = np.ma.masked_invalid(from_shaped_array(nexus_tile.tile.grid_tile.variable_data)) + self.assertEqual(3306, np.ma.count(tile_data)) + + # Check meta data + meta_list = nexus_tile.tile.grid_tile.meta_data + self.assertEqual(3, len(meta_list)) + wind_dir = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_dir') + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape) + self.assertIsNotNone(wind_dir) + wind_speed = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'wind_speed') + self.assertIsNotNone(wind_speed) + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_speed.meta_data)).shape) + wind_v = next(meta_obj for meta_obj in meta_list if meta_obj.name == 'vwnd') + self.assertIsNotNone(wind_v) + self.assertEqual(tile_data.shape, np.ma.masked_invalid(from_shaped_array(wind_v.meta_data)).shape) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/convert_iceshelf.py ---------------------------------------------------------------------- diff --git a/tests/convert_iceshelf.py b/tests/convert_iceshelf.py new file mode 100644 index 0000000..cf556dd --- /dev/null +++ b/tests/convert_iceshelf.py @@ -0,0 +1,78 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + +import datetime + +import netCDF4 +import numpy as np + +time_units = 'days since 1981-01-01 00:00:00' +time_units_attr_name = 'units' +time_calendar = 'standard' +time_calendar_attr_name = 'calendar' +time_var_name = 'time' +lat_var_name = 'lat' +lon_var_name = 'lon' + +phony_dimension_map = { + 'phony_dim_0': time_var_name, + 'phony_dim_1': lat_var_name, + 'phony_dim_2': lon_var_name +} + + +def float_to_datetime(time_float): + """ + Convert time_float (a float in the form of 4-digit_year.fractional year eg. 1994.0384) to a datetime object + """ + year = int(time_float) + remainder = time_float - year + beginning_of_year = datetime.datetime(year, 1, 1) + end_of_year = datetime.datetime(year + 1, 1, 1) + seconds = remainder * (end_of_year - beginning_of_year).total_seconds() + return beginning_of_year + datetime.timedelta(seconds=seconds) + + +with netCDF4.Dataset('/Users/greguska/data/ice_shelf_dh/ice_shelf_dh_v1.h5') as input_ds: + latitudes_1d = input_ds[lat_var_name][:, 0] + longitudes_1d = input_ds[lon_var_name][0, :] + + times_as_int = np.fromiter( + (netCDF4.date2num(float_to_datetime(time), time_units, calendar=time_calendar) for time in + input_ds[time_var_name][:]), + dtype=str(input_ds[time_var_name].dtype), count=len(input_ds[time_var_name][:])) + + with netCDF4.Dataset('/Users/greguska/data/ice_shelf_dh/ice_shelf_dh_v1.nc', mode='w') as output_ds: + output_ds.setncatts({att: input_ds.getncattr(att) for att in input_ds.ncattrs()}) + + for in_dimension_name in input_ds.dimensions: + out_dimension_name = phony_dimension_map[in_dimension_name] + output_ds.createDimension(out_dimension_name, len(input_ds.dimensions[in_dimension_name])) + + for in_variable_name, in_variable in input_ds.variables.iteritems(): + + if in_variable_name in [time_var_name, lat_var_name, lon_var_name]: + output_ds.createVariable(in_variable_name, in_variable.dtype, (in_variable_name,)) + else: + output_ds.createVariable(in_variable_name, in_variable.dtype, + tuple([phony_dimension_map[dim] for dim in in_variable.dimensions])) + + for attr_name in in_variable.ncattrs(): + attr_value = in_variable.getncattr(attr_name) + if isinstance(attr_value, list): + output_ds[in_variable_name].setncattr_string(attr_name, attr_value) + else: + output_ds[in_variable_name].setncattr(attr_name, attr_value) + + if in_variable_name == lat_var_name: + output_ds[in_variable_name][:] = latitudes_1d + elif in_variable_name == lon_var_name: + output_ds[in_variable_name][:] = longitudes_1d + elif in_variable_name == time_var_name: + output_ds[in_variable_name].setncattr(time_calendar_attr_name, time_calendar) + output_ds[in_variable_name].setncattr(time_units_attr_name, time_units) + output_ds[in_variable_name][:] = times_as_int + else: + output_ds[in_variable_name][:] = input_ds[in_variable_name][:] http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/empty_mur.nc4 ---------------------------------------------------------------------- diff --git a/tests/datafiles/empty_mur.nc4 b/tests/datafiles/empty_mur.nc4 new file mode 100644 index 0000000..f65c808 Binary files /dev/null and b/tests/datafiles/empty_mur.nc4 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/not_empty_ascatb.nc4 ---------------------------------------------------------------------- diff --git a/tests/datafiles/not_empty_ascatb.nc4 b/tests/datafiles/not_empty_ascatb.nc4 new file mode 100644 index 0000000..d8ef90b Binary files /dev/null and b/tests/datafiles/not_empty_ascatb.nc4 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/not_empty_avhrr.nc4 ---------------------------------------------------------------------- diff --git a/tests/datafiles/not_empty_avhrr.nc4 b/tests/datafiles/not_empty_avhrr.nc4 new file mode 100644 index 0000000..af24071 Binary files /dev/null and b/tests/datafiles/not_empty_avhrr.nc4 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/not_empty_ccmp.nc ---------------------------------------------------------------------- diff --git a/tests/datafiles/not_empty_ccmp.nc b/tests/datafiles/not_empty_ccmp.nc new file mode 100644 index 0000000..b7b491d Binary files /dev/null and b/tests/datafiles/not_empty_ccmp.nc differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/not_empty_measures_alt.nc ---------------------------------------------------------------------- diff --git a/tests/datafiles/not_empty_measures_alt.nc b/tests/datafiles/not_empty_measures_alt.nc new file mode 100644 index 0000000..fd03c6d Binary files /dev/null and b/tests/datafiles/not_empty_measures_alt.nc differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/not_empty_mur.nc4 ---------------------------------------------------------------------- diff --git a/tests/datafiles/not_empty_mur.nc4 b/tests/datafiles/not_empty_mur.nc4 new file mode 100644 index 0000000..09d31fd Binary files /dev/null and b/tests/datafiles/not_empty_mur.nc4 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/not_empty_smap.h5 ---------------------------------------------------------------------- diff --git a/tests/datafiles/not_empty_smap.h5 b/tests/datafiles/not_empty_smap.h5 new file mode 100644 index 0000000..956cbc5 Binary files /dev/null and b/tests/datafiles/not_empty_smap.h5 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/not_empty_wswm.nc ---------------------------------------------------------------------- diff --git a/tests/datafiles/not_empty_wswm.nc b/tests/datafiles/not_empty_wswm.nc new file mode 100644 index 0000000..772bbcb Binary files /dev/null and b/tests/datafiles/not_empty_wswm.nc differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/datafiles/partial_empty_mur.nc4 ---------------------------------------------------------------------- diff --git a/tests/datafiles/partial_empty_mur.nc4 b/tests/datafiles/partial_empty_mur.nc4 new file mode 100644 index 0000000..d95a5dc Binary files /dev/null and b/tests/datafiles/partial_empty_mur.nc4 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/dumped_nexustiles/ascat_longitude_more_than_180.bin ---------------------------------------------------------------------- diff --git a/tests/dumped_nexustiles/ascat_longitude_more_than_180.bin b/tests/dumped_nexustiles/ascat_longitude_more_than_180.bin new file mode 100644 index 0000000..75e3374 Binary files /dev/null and b/tests/dumped_nexustiles/ascat_longitude_more_than_180.bin differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/dumped_nexustiles/ascatb_nonempty_nexustile.bin ---------------------------------------------------------------------- diff --git a/tests/dumped_nexustiles/ascatb_nonempty_nexustile.bin b/tests/dumped_nexustiles/ascatb_nonempty_nexustile.bin new file mode 100644 index 0000000..9ace5ea Binary files /dev/null and b/tests/dumped_nexustiles/ascatb_nonempty_nexustile.bin differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/dumped_nexustiles/avhrr_nonempty_nexustile.bin ---------------------------------------------------------------------- diff --git a/tests/dumped_nexustiles/avhrr_nonempty_nexustile.bin b/tests/dumped_nexustiles/avhrr_nonempty_nexustile.bin new file mode 100644 index 0000000..c21398d Binary files /dev/null and b/tests/dumped_nexustiles/avhrr_nonempty_nexustile.bin differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/dumped_nexustiles/ccmp_nonempty_nexustile.bin ---------------------------------------------------------------------- diff --git a/tests/dumped_nexustiles/ccmp_nonempty_nexustile.bin b/tests/dumped_nexustiles/ccmp_nonempty_nexustile.bin new file mode 100644 index 0000000..1c1fd7f Binary files /dev/null and b/tests/dumped_nexustiles/ccmp_nonempty_nexustile.bin differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/dumped_nexustiles/smap_nonempty_nexustile.bin ---------------------------------------------------------------------- diff --git a/tests/dumped_nexustiles/smap_nonempty_nexustile.bin b/tests/dumped_nexustiles/smap_nonempty_nexustile.bin new file mode 100644 index 0000000..d19301a Binary files /dev/null and b/tests/dumped_nexustiles/smap_nonempty_nexustile.bin differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/hd5splitter.py ---------------------------------------------------------------------- diff --git a/tests/hd5splitter.py b/tests/hd5splitter.py new file mode 100644 index 0000000..c2a6655 --- /dev/null +++ b/tests/hd5splitter.py @@ -0,0 +1,123 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + + +def hd5_copy(source, dest): + for key in source.keys(): + source.copy('/' + key, dest['/'], name=key) + + print(key) + + if str(key) == 'time': + dest[key + '_c'] = dest[key][0:4] + elif str(key) == 'longitude': + dest[key + '_c'] = dest[key][0:87] + elif str(key) == 'latitude': + dest[key + '_c'] = dest[key][0:38] + else: + dest[key + '_c'] = dest[key][0:4, 0:38, 0:87] + + # Useful for swath data: + # if dest[key].ndim == 2: + # dest[key + '_c'] = dest[key][0:76, 181:183] + # elif dest[key].ndim == 3: + # dest[key + '_c'] = dest[key][0:76, 181:183, :] + # elif dest[key].ndim == 1: + # dest[key + '_c'] = dest[key][181:183] + + for att in dest[key + '_c'].attrs: + try: + dest[key + '_c'].attrs.modify(dest[key].attrs.get(att, default="")) + except IOError: + print("error " + att) + pass + dest[key + '_c'].attrs.update(dest[key].attrs) + del dest[key] + dest[key] = dest[key + '_c'] + del dest[key + '_c'] + + print(dest[key]) + + for att in dest.attrs: + try: + dest.attrs.modify(source.attrs.get(att, default="")) + except IOError: + print("error " + att) + pass + + # dest.attrs.update(source.attrs) + + dest.flush() + + +def netcdf_subset(source, dest): + dtime = dest.createDimension(dimname=TIME, size=TIME_SLICE.stop - TIME_SLICE.start) + # dlat = dest.createDimension(dimname=LATITUDE, size=LATITUDE_SLICE.stop - LATITUDE_SLICE.start) + # dlon = dest.createDimension(dimname=LONGITUDE, size=LONGITUDE_SLICE.stop - LONGITUDE_SLICE.start) + drivid = dest.createDimension(dimname='rivid', size=LONGITUDE_SLICE.stop - LONGITUDE_SLICE.start) + + dest.setncatts(source.__dict__) + + for variable in [v for v in source.variables if v in ['Qout', TIME, LONGITUDE, LATITUDE]]: + variable = source[variable] + + if variable.name == TIME: + dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype, dimensions=(dtime.name,)) + dest[variable.name].setncatts(variable.__dict__) + dvar[:] = variable[TIME_SLICE] + elif variable.name == LONGITUDE: + dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype, dimensions=(drivid.name,)) + dest[variable.name].setncatts(variable.__dict__) + dvar[:] = variable[LONGITUDE_SLICE] + elif variable.name == LATITUDE: + dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype, dimensions=(drivid.name,)) + dest[variable.name].setncatts(variable.__dict__) + dvar[:] = variable[LATITUDE_SLICE] + else: + dvar = dest.createVariable(varname=variable.name, datatype=variable.dtype, + dimensions=(dtime.name, drivid.name)) + dest[variable.name].setncatts(variable.__dict__) + dvar[:] = variable[TIME_SLICE, LONGITUDE_SLICE] + + dest.sync() + dest.close() + + +from netCDF4 import Dataset + +LATITUDE = 'lat' +LATITUDE_SLICE = slice(0, 1000) +LONGITUDE = 'lon' +LONGITUDE_SLICE = slice(0, 1000) +TIME = 'time' +TIME_SLICE = slice(0, 1) + +hinput = Dataset( + '/Users/greguska/data/swot_example/latest/Qout_WSWM_729days_p0_dtR900s_n1_preonly_20160416.nc', + 'r') +houtput = Dataset( + '/Users/greguska/data/swot_example/latest/Qout_WSWM_729days_p0_dtR900s_n1_preonly_20160416.split.nc', + mode='w') + +netcdf_subset(hinput, houtput) + +# # from h5py import File, Dataset +# hinput = File( +# '/Users/greguska/githubprojects/nexus/nexus-ingest/developer-box/data/ccmp/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.nc', +# 'r') +# houput = File( +# '/Users/greguska/githubprojects/nexus/nexus-ingest/developer-box/data/ccmp/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.split.nc', +# 'w') + +# hd5_copy(hinput, houput) + +# print hinput['/'] +# print houtput['/'] + +# print [attr for attr in hinput.attrs] +# print [attr for attr in houtput.attrs] + +# hinput.close() +# houtput.close() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/kelvintocelsius_test.py ---------------------------------------------------------------------- diff --git a/tests/kelvintocelsius_test.py b/tests/kelvintocelsius_test.py new file mode 100644 index 0000000..182abc7 --- /dev/null +++ b/tests/kelvintocelsius_test.py @@ -0,0 +1,40 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import unittest +from os import path + +import nexusproto.NexusContent_pb2 as nexusproto +import numpy as np +from nexusproto.serialization import from_shaped_array + +import processors + + +class TestAvhrrData(unittest.TestCase): + def setUp(self): + self.module = processors.KelvinToCelsius() + + def test_kelvin_to_celsius(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'avhrr_nonempty_nexustile.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str) + sst_before = from_shaped_array(nexus_tile_before.tile.grid_tile.variable_data) + + results = list(self.module.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile_after = results[0] + sst_after = from_shaped_array(nexus_tile_after.tile.grid_tile.variable_data) + + # Just spot check a couple of values + expected_sst = np.subtract(sst_before[0][0][0], np.float32(273.15)) + self.assertEqual(expected_sst, sst_after[0][0][0]) + + expected_sst = np.subtract(sst_before[0][9][9], np.float32(273.15)) + self.assertEqual(expected_sst, sst_after[0][9][9]) http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/processorchain_test.py ---------------------------------------------------------------------- diff --git a/tests/processorchain_test.py b/tests/processorchain_test.py new file mode 100644 index 0000000..3563a1b --- /dev/null +++ b/tests/processorchain_test.py @@ -0,0 +1,92 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import unittest +from os import path + +from processors.processorchain import ProcessorChain + + +class TestRunChainMethod(unittest.TestCase): + def test_run_chain_read_filter_all(self): + processor_list = [ + {'name': 'GridReadingProcessor', + 'config': {'latitude': 'lat', + 'longitude': 'lon', + 'time': 'time', + 'variable_to_read': 'analysed_sst'}}, + {'name': 'EmptyTileFilter'} + ] + processorchain = ProcessorChain(processor_list) + + test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4') + + gen = processorchain.process("time:0:1,lat:0:1,lon:0:1;time:0:1,lat:1:2,lon:0:1;file://%s" % test_file) + for message in gen: + self.fail("Should not produce any messages. Message: %s" % message) + + def test_run_chain_read_filter_none(self): + processor_list = [ + {'name': 'GridReadingProcessor', + 'config': {'latitude': 'lat', + 'longitude': 'lon', + 'time': 'time', + 'variable_to_read': 'analysed_sst'}}, + {'name': 'EmptyTileFilter'} + ] + processorchain = ProcessorChain(processor_list) + + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_mur.nc4') + + results = list( + processorchain.process("time:0:1,lat:0:1,lon:0:1;time:0:1,lat:1:2,lon:0:1;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + def test_run_chain_read_filter_kelvin_summarize(self): + processor_list = [ + {'name': 'GridReadingProcessor', + 'config': {'latitude': 'lat', + 'longitude': 'lon', + 'time': 'time', + 'variable_to_read': 'analysed_sst'}}, + {'name': 'EmptyTileFilter'}, + {'name': 'KelvinToCelsius'}, + {'name': 'TileSummarizingProcessor'} + ] + processorchain = ProcessorChain(processor_list) + + test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_mur.nc4') + + results = list( + processorchain.process("time:0:1,lat:0:1,lon:0:1;time:0:1,lat:1:2,lon:0:1;file://%s" % test_file)) + + self.assertEqual(2, len(results)) + + def test_run_chain_partial_empty(self): + processor_list = [ + {'name': 'GridReadingProcessor', + 'config': {'latitude': 'lat', + 'longitude': 'lon', + 'time': 'time', + 'variable_to_read': 'analysed_sst'}}, + {'name': 'EmptyTileFilter'}, + {'name': 'KelvinToCelsius'}, + {'name': 'TileSummarizingProcessor'} + ] + processorchain = ProcessorChain(processor_list) + + test_file = path.join(path.dirname(__file__), 'datafiles', 'partial_empty_mur.nc4') + + results = list( + processorchain.process("time:0:1,lat:0:10,lon:0:10;time:0:1,lat:489:499,lon:0:10;file://%s" % test_file)) + + self.assertEqual(1, len(results)) + tile = results[0] + + self.assertTrue(tile.summary.HasField('bbox'), "bbox is missing") + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/regrid1x1_test.py ---------------------------------------------------------------------- diff --git a/tests/regrid1x1_test.py b/tests/regrid1x1_test.py new file mode 100644 index 0000000..b080989 --- /dev/null +++ b/tests/regrid1x1_test.py @@ -0,0 +1,79 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import os +import unittest + +import processors + + +def delete_file_if_exists(filename): + try: + os.remove(filename) + except OSError: + pass + + +class TestSSHData(unittest.TestCase): + + def setUp(self): + self.test_file = os.path.join(os.path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc') + self.prefix = 'test-regrid' + self.expected_output_path = os.path.join(os.path.dirname(self.test_file), + self.prefix + os.path.basename(self.test_file)) + delete_file_if_exists(self.expected_output_path) + + def tearDown(self): + delete_file_if_exists(self.expected_output_path) + + def test_ssh_grid(self): + regridder = processors.Regrid1x1('SLA', 'Latitude', 'Longitude', 'Time', + variable_valid_range='SLA:-100.0:100.0:SLA_ERR:-5000:5000', + filename_prefix=self.prefix) + + results = list(regridder.process(self.test_file)) + + self.assertEqual(1, len(results)) + + +class TestGRACEData(unittest.TestCase): + def setUp(self): + self.test_file = '' # os.path.join(os.path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc') + self.prefix = 'test-regrid' + self.expected_output_path = os.path.join(os.path.dirname(self.test_file), + self.prefix + os.path.basename(self.test_file)) + delete_file_if_exists(self.expected_output_path) + + def tearDown(self): + delete_file_if_exists(self.expected_output_path) + + @unittest.skip + def test_lwe_grid(self): + regridder = processors.Regrid1x1('lwe_thickness', 'lat', 'lon', 'tim', + filename_prefix=self.prefix) + + results = list(regridder.process(self.test_file)) + + self.assertEqual(1, len(results)) + + +class TestIceShelfData(unittest.TestCase): + def setUp(self): + self.test_file = '' # os.path.join(os.path.dirname(__file__), 'datafiles', 'not_empty_measures_alt.nc') + self.prefix = 'test-regrid' + self.expected_output_path = os.path.join(os.path.dirname(self.test_file), + self.prefix + os.path.basename(self.test_file)) + delete_file_if_exists(self.expected_output_path) + + def tearDown(self): + delete_file_if_exists(self.expected_output_path) + + @unittest.skip + def test_height_raw(self): + regridder = processors.Regrid1x1('height_raw,height_filt,height_err', 'lat', 'lon', 'tim', + filename_prefix=self.prefix) + + results = list(regridder.process(self.test_file)) + + self.assertEqual(1, len(results)) http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningesterpy/blob/df17d945/tests/subtract180longitude_test.py ---------------------------------------------------------------------- diff --git a/tests/subtract180longitude_test.py b/tests/subtract180longitude_test.py new file mode 100644 index 0000000..5ad55c2 --- /dev/null +++ b/tests/subtract180longitude_test.py @@ -0,0 +1,57 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import unittest +from os import path + +import nexusproto.NexusContent_pb2 as nexusproto +import numpy as np +from nexusproto.serialization import from_shaped_array + +import processors + + +class TestAscatbUData(unittest.TestCase): + + def test_subtraction_longitudes_less_than_180(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascatb_nonempty_nexustile.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str) + longitudes_before = from_shaped_array(nexus_tile_before.tile.swath_tile.longitude) + + subtract = processors.Subtract180Longitude() + + results = list(subtract.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile_after = results[0] + longitudes_after = from_shaped_array(nexus_tile_after.tile.swath_tile.longitude) + + self.assertTrue(np.all(np.equal(longitudes_before, longitudes_after))) + + def test_subtraction_longitudes_greater_than_180(self): + test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascat_longitude_more_than_180.bin') + + with open(test_file, 'rb') as f: + nexustile_str = f.read() + + nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str) + longitudes_before = from_shaped_array(nexus_tile_before.tile.swath_tile.longitude) + + subtract = processors.Subtract180Longitude() + + results = list(subtract.process(nexustile_str)) + + self.assertEqual(1, len(results)) + + nexus_tile_after = results[0] + longitudes_after = from_shaped_array(nexus_tile_after.tile.swath_tile.longitude) + + self.assertTrue(np.all(np.not_equal(longitudes_before, longitudes_after))) + self.assertTrue(np.all(longitudes_after[longitudes_after < 0])) + self.assertAlmostEqual(-96.61, longitudes_after[0][26], places=2)
