This is an automated email from the ASF dual-hosted git repository. fgreg pushed a commit to branch v1.0.0-rc1 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ningesterpy.git
commit 84557068dfa863efec72de310f94ab7c84e3cf35 Author: Frank Greguska <[email protected]> AuthorDate: Mon Mar 12 18:29:12 2018 -0700 update processor chain to correctly parse list based configurations --- sdap/processors/processorchain.py | 24 +++++++++++- tests/processorchain_test.py | 81 +++++++++++++++++++++++++++++++++------ 2 files changed, 91 insertions(+), 14 deletions(-) diff --git a/sdap/processors/processorchain.py b/sdap/processors/processorchain.py index 6a68b04..82bad8e 100644 --- a/sdap/processors/processorchain.py +++ b/sdap/processors/processorchain.py @@ -14,6 +14,7 @@ # limitations under the License. import inspect +import re import sdap.processors @@ -51,18 +52,37 @@ class ProcessorChain(sdap.processors.Processor): except KeyError as e: raise ProcessorNotFound(processor['name']) from e + processor_config = dict(**processor['config']) + missing_args = [] for arg in inspect.signature(processor_constructor).parameters.keys(): if arg in ['args', 'kwargs']: continue - if arg not in processor['config']: + if arg not in processor_config: missing_args.append(arg) + # Need to check for list type args + list_pattern = re.compile('\.\d+$') + list_args = [k for k in processor_config if list_pattern.search(k)] + if list_args: + import itertools + grouped = itertools.groupby(list_args, key=lambda k: k.split('.')[0]) + for group, grouped_args in grouped: + for list_arg in grouped_args: + key, idx = list_arg.split('.') + if group not in processor_config: + processor_config[group] = [] + + processor_config[group].insert(int(idx), processor_config[list_arg]) + del(processor_config[list_arg]) + + # Check if the list args satisfied the + missing_args = list(filter(lambda a: a not in processor_config.keys(), missing_args)) if missing_args: raise MissingProcessorArguments(processor['name'], missing_args) if 'config' in processor.keys(): - processor_instance = processor_constructor(**processor['config']) + processor_instance = processor_constructor(**processor_config) else: processor_instance = processor_constructor() diff --git a/tests/processorchain_test.py b/tests/processorchain_test.py index d7b5dfa..26f2e52 100644 --- a/tests/processorchain_test.py +++ b/tests/processorchain_test.py @@ -21,6 +21,63 @@ from nexusproto import DataTile_pb2 as nexusproto from sdap.processors.processorchain import ProcessorChain +class TestConstructChain(unittest.TestCase): + + def test_construct_chain_with_list_config(self): + processor_list = [ + {'name': 'TimeSeriesReadingProcessor', + 'config': {'latitude': 'lat', + 'longitude': 'lon', + 'time': 'time', + 'variable_to_read': 'Qout'}}, + {'name': 'EmptyTileFilter', 'config': {}}, + {'name': 'PromoteVariableToGlobalAttribute', + 'config': { + 'attribute_name': 'rivid_i', + 'variable_name': 'rivid', + 'dimensioned_by.0': 'rivid', + 'dimensioned_by.1': 'other' + }}, + {'name': 'TileSummarizingProcessor', 'config': {}} + ] + + processorchain = ProcessorChain(processor_list) + + self.assertIsNotNone(processorchain) + + def test_construct_chain_with_multiple_list_config(self): + processor_list = [ + {'name': 'PromoteVariableToGlobalAttribute', + 'config': { + 'attribute_name': 'rivid_i', + 'variable_name': 'rivid', + 'dimensioned_by.0': 'rivid', + 'dimensioned_by.1': 'other', + 'unused.0': 'list', + 'unused.1': 'second' + }} + ] + + processorchain = ProcessorChain(processor_list) + + self.assertIsNotNone(processorchain) + + def test_construct_chain_with_list_config_bad_index(self): + processor_list = [ + {'name': 'PromoteVariableToGlobalAttribute', + 'config': { + 'attribute_name': 'rivid_i', + 'variable_name': 'rivid', + 'dimensioned_by.0': 'rivid', + 'dimensioned_by.10': 'other' + }} + ] + + processorchain = ProcessorChain(processor_list) + + self.assertIsNotNone(processorchain) + + class TestRunChainMethod(unittest.TestCase): def test_run_chain_read_filter_all(self): processor_list = [ @@ -29,7 +86,7 @@ class TestRunChainMethod(unittest.TestCase): 'longitude': 'lon', 'time': 'time', 'variable_to_read': 'analysed_sst'}}, - {'name': 'EmptyTileFilter'} + {'name': 'EmptyTileFilter', 'config': {}} ] processorchain = ProcessorChain(processor_list) @@ -52,7 +109,7 @@ class TestRunChainMethod(unittest.TestCase): 'longitude': 'lon', 'time': 'time', 'variable_to_read': 'analysed_sst'}}, - {'name': 'EmptyTileFilter'} + {'name': 'EmptyTileFilter', 'config': {}} ] processorchain = ProcessorChain(processor_list) @@ -75,9 +132,9 @@ class TestRunChainMethod(unittest.TestCase): 'longitude': 'lon', 'time': 'time', 'variable_to_read': 'analysed_sst'}}, - {'name': 'EmptyTileFilter'}, - {'name': 'KelvinToCelsius'}, - {'name': 'TileSummarizingProcessor'} + {'name': 'EmptyTileFilter', 'config': {}}, + {'name': 'KelvinToCelsius', 'config': {}}, + {'name': 'TileSummarizingProcessor', 'config': {}} ] processorchain = ProcessorChain(processor_list) @@ -100,9 +157,9 @@ class TestRunChainMethod(unittest.TestCase): 'longitude': 'lon', 'time': 'time', 'variable_to_read': 'analysed_sst'}}, - {'name': 'EmptyTileFilter'}, - {'name': 'KelvinToCelsius'}, - {'name': 'TileSummarizingProcessor'} + {'name': 'EmptyTileFilter', 'config': {}}, + {'name': 'KelvinToCelsius', 'config': {}}, + {'name': 'TileSummarizingProcessor', 'config': {}} ] processorchain = ProcessorChain(processor_list) @@ -137,15 +194,15 @@ class TestRunChainMethod(unittest.TestCase): 'longitude': 'lon', 'time': 'time', 'variable_to_read': 'analysed_sst'}}, - {'name': 'EmptyTileFilter'}, - {'name': 'KelvinToCelsius'}, + {'name': 'EmptyTileFilter', 'config': {}}, + {'name': 'KelvinToCelsius', 'config': {}}, {'name': 'PromoteVariableToGlobalAttribute', 'config': { 'attribute_name': 'time_i', 'variable_name': 'time', - 'dimensioned_by': ['time'] + 'dimensioned_by.0': 'time' }}, - {'name': 'TileSummarizingProcessor'} + {'name': 'TileSummarizingProcessor', 'config': {}} ] processorchain = ProcessorChain(processor_list)
