This is an automated email from the ASF dual-hosted git repository. fgreg pushed a commit to branch v1.0.0-rc1 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ningesterpy.git
commit 49783072aaaf18fa75392472acf7140cc02efa1f Author: Frank Greguska <[email protected]> AuthorDate: Fri Jan 26 19:42:16 2018 -0800 added ningesterpy everywhere --- .gitignore | 1 - ningesterpy/ningesterpy.py | 2 +- ningesterpy/processors/__init__.py | 22 +++++++++++----------- ningesterpy/processors/callncpdq.py | 2 +- ningesterpy/processors/callncra.py | 2 +- ningesterpy/processors/computespeeddirfromuv.py | 2 +- ningesterpy/processors/emptytilefilter.py | 2 +- ningesterpy/processors/kelvintocelsius.py | 2 +- .../processors/normalizetimebeginningofmonth.py | 2 +- ningesterpy/processors/processorchain.py | 6 +++--- ningesterpy/processors/regrid1x1.py | 2 +- ningesterpy/processors/subtract180longitude.py | 2 +- ningesterpy/processors/tilereadingprocessor.py | 2 +- ningesterpy/processors/tilesummarizingprocessor.py | 2 +- ningesterpy/processors/winddirspeedtouv.py | 2 +- scripts/__init__.py | 0 {tests => scripts}/convert_iceshelf.py | 0 {tests => scripts}/hd5splitter.py | 0 setup.py | 2 +- tests/callncpdq_test.py | 2 +- tests/computespeeddirfromuv_test.py | 6 +++--- tests/kelvintocelsius_test.py | 4 ++-- tests/processorchain_test.py | 2 +- tests/regrid1x1_test.py | 8 ++++---- tests/subtract180longitude_test.py | 6 +++--- tests/tilereadingprocessor_test.py | 16 ++++++++-------- tests/tilesumarizingprocessor_test.py | 6 +++--- tests/winddirspeedtouv_test.py | 6 +++--- 28 files changed, 55 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 788c98f..cc74a02 100644 --- a/.gitignore +++ b/.gitignore @@ -112,7 +112,6 @@ venv.bak/ [Ll]ib [Ll]ib64 [Ll]ocal -[Ss]cripts pyvenv.cfg pip-selfcheck.json ### JetBrains template diff --git a/ningesterpy/ningesterpy.py b/ningesterpy/ningesterpy.py index e3f1d02..17eceba 100644 --- a/ningesterpy/ningesterpy.py +++ b/ningesterpy/ningesterpy.py @@ -24,7 +24,7 @@ from google.protobuf.json_format import ParseError from werkzeug.exceptions import HTTPException, BadRequest from werkzeug.exceptions import default_exceptions -from processors.processorchain import ProcessorChain, ProcessorNotFound, MissingProcessorArguments +from ningesterpy.processors.processorchain import ProcessorChain, ProcessorNotFound, MissingProcessorArguments applog = logging.getLogger(__name__) app = Flask(__name__) diff --git a/ningesterpy/processors/__init__.py b/ningesterpy/processors/__init__.py index e837ce5..8902ff6 100644 --- a/ningesterpy/processors/__init__.py +++ b/ningesterpy/processors/__init__.py @@ -49,17 +49,17 @@ class NexusTileProcessor(Processor): # All installed processors need to be imported and added to the dict below -from processors.callncpdq import CallNcpdq -from processors.callncra import CallNcra -from processors.computespeeddirfromuv import ComputeSpeedDirFromUV -from processors.emptytilefilter import EmptyTileFilter -from processors.kelvintocelsius import KelvinToCelsius -from processors.normalizetimebeginningofmonth import NormalizeTimeBeginningOfMonth -from processors.regrid1x1 import Regrid1x1 -from processors.subtract180longitude import Subtract180Longitude -from processors.tilereadingprocessor import GridReadingProcessor, SwathReadingProcessor, TimeSeriesReadingProcessor -from processors.tilesummarizingprocessor import TileSummarizingProcessor -from processors.winddirspeedtouv import WindDirSpeedToUV +from ningesterpy.processors.callncpdq import CallNcpdq +from ningesterpy.processors.callncra import CallNcra +from ningesterpy.processors.computespeeddirfromuv import ComputeSpeedDirFromUV +from ningesterpy.processors.emptytilefilter import EmptyTileFilter +from ningesterpy.processors.kelvintocelsius import KelvinToCelsius +from ningesterpy.processors.normalizetimebeginningofmonth import NormalizeTimeBeginningOfMonth +from ningesterpy.processors.regrid1x1 import Regrid1x1 +from ningesterpy.processors.subtract180longitude import Subtract180Longitude +from ningesterpy.processors.tilereadingprocessor import GridReadingProcessor, SwathReadingProcessor, TimeSeriesReadingProcessor +from ningesterpy.processors.tilesummarizingprocessor import TileSummarizingProcessor +from ningesterpy.processors.winddirspeedtouv import WindDirSpeedToUV INSTALLED_PROCESSORS = { "CallNcpdq": CallNcpdq, diff --git a/ningesterpy/processors/callncpdq.py b/ningesterpy/processors/callncpdq.py index 315a9c8..3dfa9b8 100644 --- a/ningesterpy/processors/callncpdq.py +++ b/ningesterpy/processors/callncpdq.py @@ -18,7 +18,7 @@ import logging import os from subprocess import call -from processors import Processor +from ningesterpy.processors import Processor class CallNcpdq(Processor): diff --git a/ningesterpy/processors/callncra.py b/ningesterpy/processors/callncra.py index 4276995..b5a8e2e 100644 --- a/ningesterpy/processors/callncra.py +++ b/ningesterpy/processors/callncra.py @@ -20,7 +20,7 @@ from subprocess import call from netCDF4 import Dataset, num2date -from processors import Processor +from ningesterpy.processors import Processor class CallNcra(Processor): diff --git a/ningesterpy/processors/computespeeddirfromuv.py b/ningesterpy/processors/computespeeddirfromuv.py index 57a3b78..9a53cba 100644 --- a/ningesterpy/processors/computespeeddirfromuv.py +++ b/ningesterpy/processors/computespeeddirfromuv.py @@ -17,7 +17,7 @@ import numpy from nexusproto.serialization import from_shaped_array, to_shaped_array -from processors import NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor def calculate_speed_direction(wind_u, wind_v): diff --git a/ningesterpy/processors/emptytilefilter.py b/ningesterpy/processors/emptytilefilter.py index 976b8c3..0641f38 100644 --- a/ningesterpy/processors/emptytilefilter.py +++ b/ningesterpy/processors/emptytilefilter.py @@ -19,7 +19,7 @@ from nexusproto import DataTile_pb2 as nexusproto import numpy from nexusproto.serialization import from_shaped_array -from processors import NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor logger = logging.getLogger('emptytilefilter') diff --git a/ningesterpy/processors/kelvintocelsius.py b/ningesterpy/processors/kelvintocelsius.py index 5209798..82d1129 100644 --- a/ningesterpy/processors/kelvintocelsius.py +++ b/ningesterpy/processors/kelvintocelsius.py @@ -16,7 +16,7 @@ from nexusproto.serialization import from_shaped_array, to_shaped_array -from processors import NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor class KelvinToCelsius(NexusTileProcessor): diff --git a/ningesterpy/processors/normalizetimebeginningofmonth.py b/ningesterpy/processors/normalizetimebeginningofmonth.py index ef452ea..0889117 100644 --- a/ningesterpy/processors/normalizetimebeginningofmonth.py +++ b/ningesterpy/processors/normalizetimebeginningofmonth.py @@ -17,7 +17,7 @@ import datetime from pytz import timezone -from processors import NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1)) diff --git a/ningesterpy/processors/processorchain.py b/ningesterpy/processors/processorchain.py index 55a3e19..b236f4b 100644 --- a/ningesterpy/processors/processorchain.py +++ b/ningesterpy/processors/processorchain.py @@ -15,7 +15,7 @@ import inspect -import processors +import ningesterpy.processors class BadChainException(Exception): @@ -39,7 +39,7 @@ class MissingProcessorArguments(Exception): super().__init__(message, *args) -class ProcessorChain(processors.Processor): +class ProcessorChain(ningesterpy.processors.Processor): def __init__(self, processor_list, *args, **kwargs): super().__init__(*args, **kwargs) @@ -47,7 +47,7 @@ class ProcessorChain(processors.Processor): # Attempt to construct the needed processors for processor in processor_list: try: - processor_constructor = processors.INSTALLED_PROCESSORS[processor['name']] + processor_constructor = ningesterpy.processors.INSTALLED_PROCESSORS[processor['name']] except KeyError as e: raise ProcessorNotFound(processor['name']) from e diff --git a/ningesterpy/processors/regrid1x1.py b/ningesterpy/processors/regrid1x1.py index 896f9f6..7ec8775 100644 --- a/ningesterpy/processors/regrid1x1.py +++ b/ningesterpy/processors/regrid1x1.py @@ -22,7 +22,7 @@ from netCDF4 import Dataset from pytz import timezone from scipy import interpolate -from processors import Processor +from ningesterpy.processors import Processor UTC = timezone('UTC') ISO_8601 = '%Y-%m-%dT%H:%M:%S%z' diff --git a/ningesterpy/processors/subtract180longitude.py b/ningesterpy/processors/subtract180longitude.py index 74ad846..312d1b8 100644 --- a/ningesterpy/processors/subtract180longitude.py +++ b/ningesterpy/processors/subtract180longitude.py @@ -15,7 +15,7 @@ from nexusproto.serialization import from_shaped_array, to_shaped_array -from processors import NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor class Subtract180Longitude(NexusTileProcessor): diff --git a/ningesterpy/processors/tilereadingprocessor.py b/ningesterpy/processors/tilereadingprocessor.py index 7b26c14..84cf431 100644 --- a/ningesterpy/processors/tilereadingprocessor.py +++ b/ningesterpy/processors/tilereadingprocessor.py @@ -25,7 +25,7 @@ from netCDF4 import Dataset, num2date from nexusproto.serialization import to_shaped_array, to_metadata from pytz import timezone -from processors import Processor, NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1)) diff --git a/ningesterpy/processors/tilesummarizingprocessor.py b/ningesterpy/processors/tilesummarizingprocessor.py index b0a9028..7ff8bc0 100644 --- a/ningesterpy/processors/tilesummarizingprocessor.py +++ b/ningesterpy/processors/tilesummarizingprocessor.py @@ -17,7 +17,7 @@ from nexusproto import DataTile_pb2 as nexusproto import numpy from nexusproto.serialization import from_shaped_array -from processors import NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor class NoTimeException(Exception): diff --git a/ningesterpy/processors/winddirspeedtouv.py b/ningesterpy/processors/winddirspeedtouv.py index f5d3a82..5fae135 100644 --- a/ningesterpy/processors/winddirspeedtouv.py +++ b/ningesterpy/processors/winddirspeedtouv.py @@ -20,7 +20,7 @@ from math import sin import numpy from nexusproto.serialization import from_shaped_array, to_shaped_array -from processors import NexusTileProcessor +from ningesterpy.processors import NexusTileProcessor def enum(**enums): diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert_iceshelf.py b/scripts/convert_iceshelf.py similarity index 100% rename from tests/convert_iceshelf.py rename to scripts/convert_iceshelf.py diff --git a/tests/hd5splitter.py b/scripts/hd5splitter.py similarity index 100% rename from tests/hd5splitter.py rename to scripts/hd5splitter.py diff --git a/setup.py b/setup.py index 212fa87..64a1fbc 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ setup( install_requires=pip_requirements, - packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), + packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", "scripts"]), test_suite="tests", platforms='any', diff --git a/tests/callncpdq_test.py b/tests/callncpdq_test.py index 11af9ac..f1eecc7 100644 --- a/tests/callncpdq_test.py +++ b/tests/callncpdq_test.py @@ -19,7 +19,7 @@ from os import path, remove from netCDF4 import Dataset -from processors import callncpdq +from ningesterpy.processors import callncpdq class TestMeasuresData(unittest.TestCase): diff --git a/tests/computespeeddirfromuv_test.py b/tests/computespeeddirfromuv_test.py index 16335bc..6daa962 100644 --- a/tests/computespeeddirfromuv_test.py +++ b/tests/computespeeddirfromuv_test.py @@ -19,8 +19,8 @@ from os import path import numpy as np from nexusproto.serialization import from_shaped_array -import processors -from processors.computespeeddirfromuv import calculate_speed_direction +import ningesterpy.processors +from ningesterpy.processors.computespeeddirfromuv import calculate_speed_direction class TestConversion(unittest.TestCase): @@ -86,7 +86,7 @@ class TestConversion(unittest.TestCase): class TestCcmpData(unittest.TestCase): def setUp(self): - self.module = processors.ComputeSpeedDirFromUV('uwnd', 'vwnd') + self.module = ningesterpy.processors.ComputeSpeedDirFromUV('uwnd', 'vwnd') def test_speed_dir_computation(self): test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ccmp_nonempty_nexustile.bin') diff --git a/tests/kelvintocelsius_test.py b/tests/kelvintocelsius_test.py index 2d4cd83..0d738b2 100644 --- a/tests/kelvintocelsius_test.py +++ b/tests/kelvintocelsius_test.py @@ -20,12 +20,12 @@ from nexusproto import DataTile_pb2 as nexusproto import numpy as np from nexusproto.serialization import from_shaped_array -import processors +import ningesterpy.processors class TestAvhrrData(unittest.TestCase): def setUp(self): - self.module = processors.KelvinToCelsius() + self.module = ningesterpy.processors.KelvinToCelsius() def test_kelvin_to_celsius(self): test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'avhrr_nonempty_nexustile.bin') diff --git a/tests/processorchain_test.py b/tests/processorchain_test.py index 6b47a3e..a8688d8 100644 --- a/tests/processorchain_test.py +++ b/tests/processorchain_test.py @@ -18,7 +18,7 @@ from os import path from nexusproto import DataTile_pb2 as nexusproto -from processors.processorchain import ProcessorChain +from ningesterpy.processors.processorchain import ProcessorChain class TestRunChainMethod(unittest.TestCase): diff --git a/tests/regrid1x1_test.py b/tests/regrid1x1_test.py index e05713e..45e3c0e 100644 --- a/tests/regrid1x1_test.py +++ b/tests/regrid1x1_test.py @@ -16,7 +16,7 @@ import os import unittest -import processors +import ningesterpy.processors def delete_file_if_exists(filename): @@ -39,7 +39,7 @@ class TestSSHData(unittest.TestCase): delete_file_if_exists(self.expected_output_path) def test_ssh_grid(self): - regridder = processors.Regrid1x1('SLA', 'Latitude', 'Longitude', 'Time', + regridder = ningesterpy.processors.Regrid1x1('SLA', 'Latitude', 'Longitude', 'Time', variable_valid_range='SLA:-100.0:100.0:SLA_ERR:-5000:5000', filename_prefix=self.prefix) @@ -61,7 +61,7 @@ class TestGRACEData(unittest.TestCase): @unittest.skip def test_lwe_grid(self): - regridder = processors.Regrid1x1('lwe_thickness', 'lat', 'lon', 'tim', + regridder = ningesterpy.processors.Regrid1x1('lwe_thickness', 'lat', 'lon', 'tim', filename_prefix=self.prefix) results = list(regridder.process(self.test_file)) @@ -82,7 +82,7 @@ class TestIceShelfData(unittest.TestCase): @unittest.skip def test_height_raw(self): - regridder = processors.Regrid1x1('height_raw,height_filt,height_err', 'lat', 'lon', 'tim', + regridder = ningesterpy.processors.Regrid1x1('height_raw,height_filt,height_err', 'lat', 'lon', 'tim', filename_prefix=self.prefix) results = list(regridder.process(self.test_file)) diff --git a/tests/subtract180longitude_test.py b/tests/subtract180longitude_test.py index b8794b6..e8c2f77 100644 --- a/tests/subtract180longitude_test.py +++ b/tests/subtract180longitude_test.py @@ -20,7 +20,7 @@ from nexusproto import DataTile_pb2 as nexusproto import numpy as np from nexusproto.serialization import from_shaped_array -import processors +import ningesterpy.processors class TestAscatbUData(unittest.TestCase): @@ -34,7 +34,7 @@ class TestAscatbUData(unittest.TestCase): nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str) longitudes_before = from_shaped_array(nexus_tile_before.tile.swath_tile.longitude) - subtract = processors.Subtract180Longitude() + subtract = ningesterpy.processors.Subtract180Longitude() results = list(subtract.process(nexustile_str)) @@ -54,7 +54,7 @@ class TestAscatbUData(unittest.TestCase): nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str) longitudes_before = from_shaped_array(nexus_tile_before.tile.swath_tile.longitude) - subtract = processors.Subtract180Longitude() + subtract = ningesterpy.processors.Subtract180Longitude() results = list(subtract.process(nexustile_str)) diff --git a/tests/tilereadingprocessor_test.py b/tests/tilereadingprocessor_test.py index ad31442..8b49eff 100644 --- a/tests/tilereadingprocessor_test.py +++ b/tests/tilereadingprocessor_test.py @@ -20,12 +20,12 @@ import numpy as np from nexusproto.serialization import from_shaped_array from nexusproto import DataTile_pb2 as nexusproto -import processors +import ningesterpy.processors class TestReadMurData(unittest.TestCase): def setUp(self): - self.module = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time') + self.module = ningesterpy.processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time') def test_read_empty_mur(self): test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4') @@ -82,7 +82,7 @@ class TestReadAscatbData(unittest.TestCase): def test_read_not_empty_ascatb(self): test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4') - swath_reader = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time') + swath_reader = ningesterpy.processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time') input_tile = nexusproto.NexusTile() tile_summary = nexusproto.TileSummary() @@ -115,7 +115,7 @@ class TestReadAscatbData(unittest.TestCase): test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4') - swath_reader = processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time', meta='wind_dir') + swath_reader = ningesterpy.processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time', meta='wind_dir') input_tile = nexusproto.NexusTile() tile_summary = nexusproto.TileSummary() @@ -142,7 +142,7 @@ class TestReadSmapData(unittest.TestCase): def test_read_not_empty_smap(self): test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_smap.h5') - swath_reader = processors.SwathReadingProcessor('smap_sss', 'lat', 'lon', + swath_reader = ningesterpy.processors.SwathReadingProcessor('smap_sss', 'lat', 'lon', time='row_time', glblattr_day='REV_START_TIME', glblattr_day_format='%Y-%jT%H:%M:%S.%f') @@ -187,7 +187,7 @@ class TestReadCcmpData(unittest.TestCase): def test_read_not_empty_ccmp(self): test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ccmp.nc') - ccmp_reader = processors.GridReadingProcessor('uwnd', 'latitude', 'longitude', time='time', meta='vwnd') + ccmp_reader = ningesterpy.processors.GridReadingProcessor('uwnd', 'latitude', 'longitude', time='time', meta='vwnd') input_tile = nexusproto.NexusTile() tile_summary = nexusproto.TileSummary() @@ -228,7 +228,7 @@ class TestReadAvhrrData(unittest.TestCase): def test_read_not_empty_avhrr(self): test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_avhrr.nc4') - avhrr_reader = processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time') + avhrr_reader = ningesterpy.processors.GridReadingProcessor('analysed_sst', 'lat', 'lon', time='time') input_tile = nexusproto.NexusTile() tile_summary = nexusproto.TileSummary() @@ -270,7 +270,7 @@ class TestReadWSWMData(unittest.TestCase): def test_read_not_empty_wswm(self): test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_wswm.nc') - wswm_reader = processors.TimeSeriesReadingProcessor('Qout', 'lat', 'lon', 'time') + wswm_reader = ningesterpy.processors.TimeSeriesReadingProcessor('Qout', 'lat', 'lon', 'time') input_tile = nexusproto.NexusTile() tile_summary = nexusproto.TileSummary() diff --git a/tests/tilesumarizingprocessor_test.py b/tests/tilesumarizingprocessor_test.py index 82b03c4..ebebd8b 100644 --- a/tests/tilesumarizingprocessor_test.py +++ b/tests/tilesumarizingprocessor_test.py @@ -16,7 +16,7 @@ import unittest from os import path -import processors +import ningesterpy.processors class TestSummarizeTile(unittest.TestCase): @@ -26,7 +26,7 @@ class TestSummarizeTile(unittest.TestCase): with open(test_file, 'rb') as f: nexustile_str = f.read() - summarizer = processors.TileSummarizingProcessor() + summarizer = ningesterpy.processors.TileSummarizingProcessor() results = list(summarizer.process(nexustile_str)) @@ -59,7 +59,7 @@ class TestSummarizeTile(unittest.TestCase): with open(test_file, 'rb') as f: nexustile_str = f.read() - summarizer = processors.TileSummarizingProcessor() + summarizer = ningesterpy.processors.TileSummarizingProcessor() results = list(summarizer.process(nexustile_str)) diff --git a/tests/winddirspeedtouv_test.py b/tests/winddirspeedtouv_test.py index c9b288e..4f747f9 100644 --- a/tests/winddirspeedtouv_test.py +++ b/tests/winddirspeedtouv_test.py @@ -19,7 +19,7 @@ from os import path import numpy as np from nexusproto.serialization import from_shaped_array -import processors +import ningesterpy.processors class TestAscatbUData(unittest.TestCase): @@ -30,7 +30,7 @@ class TestAscatbUData(unittest.TestCase): with open(test_file, 'rb') as f: nexustile_str = f.read() - converter = processors.WindDirSpeedToUV('U') + converter = ningesterpy.processors.WindDirSpeedToUV('U') results = list(converter.process(nexustile_str)) @@ -67,7 +67,7 @@ class TestAscatbVData(unittest.TestCase): with open(test_file, 'rb') as f: nexustile_str = f.read() - converter = processors.WindDirSpeedToUV('V') + converter = ningesterpy.processors.WindDirSpeedToUV('V') results = list(converter.process(nexustile_str))
