Repository: climate Updated Branches: refs/heads/master 7f34fc367 -> 34cadf03d
http://git-wip-us.apache.org/repos/asf/climate/blob/e926a581/ocw_config_runner/tests/test_config_parsing.py ---------------------------------------------------------------------- diff --git a/ocw_config_runner/tests/test_config_parsing.py b/ocw_config_runner/tests/test_config_parsing.py new file mode 100644 index 0000000..e918405 --- /dev/null +++ b/ocw_config_runner/tests/test_config_parsing.py @@ -0,0 +1,806 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from mock import patch +import unittest + +import configuration_parsing as parser +import ocw.metrics as metrics + +import yaml + + +class TestIsConfigValid(unittest.TestCase): + @classmethod + def setUpClass(self): + not_minimal_config = """ + datasets: + """ + self.not_minimal = yaml.load(not_minimal_config) + + not_well_formed_config = """ + datasets: + reference: + data_source: local + file_count: 1 + path: /a/fake/path/file.py + variable: pr + + targets: + - data_source: local + file_count: 5 + file_glob_pattern: something for globbing files here + variable: pr + optional_args: + name: Target1 + + - data_source: esgf + dataset_id: fake dataset id + variable: pr + esgf_username: my esgf username + esgf_password: my esgf password + + metrics: + - Bias + - TemporalStdDev + """ + self.not_well_formed = yaml.load(not_well_formed_config) + + @patch('configuration_parsing.logger') + def test_not_minimal_config(self, mock_logger): + ret = parser.is_config_valid(self.not_minimal) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Insufficient configuration file data for an evaluation' + ) + + @patch('configuration_parsing.logger') + def test_not_valid_config(self, mock_logger): + ret = parser.is_config_valid(self.not_well_formed) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Configuration data is not well formed' + ) + + +class TestValidMinimalConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + no_datasets_config = """ + metrics: + - Bias + """ + self.no_datasets = yaml.load(no_datasets_config) + + no_metrics_config = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + """ + self.no_metrics = yaml.load(no_metrics_config) + + unary_with_reference_config = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - TemporalStdDev + """ + self.unary_with_reference = yaml.load(unary_with_reference_config) + + unary_with_target_config = """ + datasets: + targets: + - data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - TemporalStdDev + """ + self.unary_with_target = yaml.load(unary_with_target_config) + + unary_no_reference_or_target = """ + datasets: + not_ref_or_target: + - data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - TemporalStdDev + """ + self.unary_no_ref_or_target = yaml.load(unary_no_reference_or_target) + + binary_valid_config = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + + targets: + - data_source: dap + url: afakeurl.com + variable: pr + metrics: + - Bias + """ + self.binary_valid = yaml.load(binary_valid_config) + + binary_no_reference_config = """ + datasets: + targets: + - data_source: dap + url: afakeurl.com + variable: pr + metrics: + - Bias + """ + self.binary_no_reference = yaml.load(binary_no_reference_config) + + binary_no_target_config = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - Bias + """ + self.binary_no_target = yaml.load(binary_no_target_config) + + @patch('configuration_parsing.logger') + def test_no_datasets(self, mock_logger): + ret = parser._valid_minimal_config(self.no_datasets) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'No datasets specified in configuration data.' + ) + + @patch('configuration_parsing.logger') + def test_no_metrics(self, mock_logger): + ret = parser._valid_minimal_config(self.no_metrics) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'No metrics specified in configuration data.' + ) + + def test_unary_with_reference(self): + ret = parser._valid_minimal_config(self.unary_with_reference) + self.assertTrue(ret) + + def test_unary_with_target(self): + ret = parser._valid_minimal_config(self.unary_with_target) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_unary_no_datasets(self, mock_logger): + ret = parser._valid_minimal_config(self.unary_no_ref_or_target) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Unary metric in configuration data requires either a reference ' + 'or target dataset to be present for evaluation. Please ensure ' + 'that your config is well formed.' + ) + + def test_valid_binary(self): + ret = parser._valid_minimal_config(self.binary_valid) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_binary_no_reference(self, mock_logger): + ret = parser._valid_minimal_config(self.binary_no_reference) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Binary metric in configuration requires both a reference ' + 'and target dataset to be present for evaluation. Please ensure ' + 'that your config is well formed.' + ) + + @patch('configuration_parsing.logger') + def test_binary_no_target(self, mock_logger): + ret = parser._valid_minimal_config(self.binary_no_target) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Binary metric in configuration requires both a reference ' + 'and target dataset to be present for evaluation. Please ensure ' + 'that your config is well formed.' + ) + + +class TestConfigIsWellFormed(unittest.TestCase): + @classmethod + def setUpClass(self): + malformed_reference_config = """ + datasets: + reference: + data_source: notavalidlocation + + metrics: + - Bias + """ + self.malformed_reference_conf = yaml.load(malformed_reference_config) + + malformed_target_list_config = """ + datasets: + targets: + notalist: + a_key: a_value + + alsonotalist: + a_key: a_value + + metrics: + - Bias + """ + self.malformed_target_list = yaml.load(malformed_target_list_config) + + missing_metric_name_config = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - NotABuiltInMetric + """ + self.missing_metric_name = yaml.load(missing_metric_name_config) + + bad_plot_config = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - Bias + + plots: + - type: NotARealPlotName + """ + bad_plot = yaml.load(bad_plot_config) + + bad_subregion_config_type = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - Bias + + subregions: + - this is a string instead of a list + """ + self.bad_subregion_type = yaml.load(bad_subregion_config_type) + + bad_subregion_config_length = """ + datasets: + reference: + data_source: dap + url: afakeurl.com + variable: pr + + metrics: + - Bias + + subregions: + - [1, 2, 3, 4, 5] + """ + self.bad_subregion_length = yaml.load(bad_subregion_config_length) + + def test_malformed_reference_config(self): + ret = parser._config_is_well_formed(self.malformed_reference_conf) + self.assertFalse(ret) + + @patch('configuration_parsing.logger') + def test_malformed_target_dataset_list(self, mock_logger): + ret = parser._config_is_well_formed(self.malformed_target_list) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + "Expected to find list of target datasets but instead found " + "object of type <type 'dict'>" + ) + + def test_not_builtin_metric(self): + ret = parser._config_is_well_formed(self.missing_metric_name) + self.assertFalse(ret) + + @patch('configuration_parsing.logger') + def test_warns_regarding_not_builtin_metric(self, mock_logger): + ret = parser._config_is_well_formed(self.missing_metric_name) + mock_logger.warn.assert_called_with( + 'Unable to locate metric name NotABuiltInMetric in built-in ' + 'metrics. If this is not a user defined metric then please check ' + 'for potential misspellings.' + ) + + def test_bad_plot_config(self): + ret = parser._config_is_well_formed(self.missing_metric_name) + self.assertFalse(ret) + + def test_bad_subregion_type(self): + ret = parser._config_is_well_formed(self.bad_subregion_type) + self.assertFalse(ret) + + def test_bad_subregion_length(self): + ret = parser._config_is_well_formed(self.bad_subregion_length) + self.assertFalse(ret) + + +class MetricFetchTest(unittest.TestCase): + @classmethod + def setUpClass(self): + binary_config = """ + metrics: + - Bias + - StdDevRatio + """ + unary_config = """ + metrics: + - TemporalStdDev + """ + self.unary_conf = yaml.load(unary_config) + self.binary_conf = yaml.load(binary_config) + + def test_contains_binary_metric(self): + ret = parser._contains_binary_metrics(self.binary_conf['metrics']) + self.assertTrue(ret) + + def test_does_not_contain_binary_metric(self): + ret = parser._contains_binary_metrics(self.unary_conf['metrics']) + self.assertFalse(ret) + + def test_contains_unary_metric(self): + ret = parser._contains_unary_metrics(self.unary_conf['metrics']) + self.assertTrue(ret) + + def test_does_not_contain_unary_metric(self): + ret = parser._contains_unary_metrics(self.binary_conf['metrics']) + self.assertFalse(ret) + + +class InvalidDatasetConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + example_config_yaml = """ + - file_count: 1 + path: /a/fake/path + variable: pr + + - data_source: invalid_location_identifier + """ + conf = yaml.load(example_config_yaml) + self.missing_data_source = conf[0] + self.invalid_data_source = conf[1] + + @patch('configuration_parsing.logger') + def test_missing_data_source_config(self, mock_logger): + parser._valid_dataset_config_data(self.missing_data_source) + mock_logger.error.assert_called_with( + 'Dataset does not contain a data_source attribute.' + ) + + @patch('configuration_parsing.logger') + def test_invalid_data_source(self, mock_logger): + parser._valid_dataset_config_data(self.invalid_data_source) + mock_logger.error.assert_called_with( + 'Dataset does not contain a valid data_source location.' + ) + + +class TestLocalDatasetConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + self.required_local_keys = set(['data_source', 'file_count', 'path', 'variable']) + example_config_yaml = """ + - data_source: local + file_count: 1 + path: /a/fake/path + variable: pr + optional_args: + name: Target1 + + - data_source: local + + - data_source: local + file_count: 5 + file_glob_pattern: something for globbing files here + variable: pr + path: /a/fake/path + optional_args: + name: Target1 + + - data_source: local + file_count: 5 + variable: pr + path: /a/fake/path + """ + + conf = yaml.load(example_config_yaml) + self.valid_local_single = conf[0] + self.invalid_local_single = conf[1] + self.valid_local_multi = conf[2] + self.invalid_local_multi = conf[1] + self.invalid_local_multi_file_glob = conf[3] + + def test_valid_local_config_single_file(self): + ret = parser._valid_dataset_config_data(self.valid_local_single) + self.assertTrue(ret) + + def test_valid_local_config_multi_file(self): + ret = parser._valid_dataset_config_data(self.valid_local_multi) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_invalid_local_config(self, mock_logger): + parser._valid_dataset_config_data(self.invalid_local_single) + + present_keys = set(self.invalid_local_single.keys()) + missing_keys = self.required_local_keys - present_keys + missing = sorted(list(missing_keys)) + + error = ( + 'Dataset does not contain required keys. ' + 'The following keys are missing: {}'.format(', '.join(missing)) + ) + mock_logger.error.assert_called_with(error) + + @patch('configuration_parsing.logger') + def test_invalid_local_config_multi_file(self, mock_logger): + # mutlifile config is handled slightly differently. We should see the + # same missing keys in this situation as we would on the single file + # local config. We will test for a missing file_glob_pattern in a + # different test. + parser._valid_dataset_config_data(self.invalid_local_multi) + + present_keys = set(self.invalid_local_multi.keys()) + missing_keys = self.required_local_keys - present_keys + missing = sorted(list(missing_keys)) + + error = ( + 'Dataset does not contain required keys. ' + 'The following keys are missing: {}'.format(', '.join(missing)) + ) + mock_logger.error.assert_called_with(error) + + @patch('configuration_parsing.logger') + def test_invalid_local_config_multi_file_missing_file_glob(self, mock_logger): + # We can't check for the file_glob_pattern pattern until after we have + # verified that the single local file config has been met. + parser._valid_dataset_config_data(self.invalid_local_multi_file_glob) + + mock_logger.error.assert_called_with( + 'Multi-file local dataset is missing key: file_glob_pattern' + ) + + +class TestRCMEDDatasetConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + self.required_rcmed_keys = set([ + 'dataset_id', + 'parameter_id', + 'min_lat', + 'max_lat', + 'min_lon', + 'max_lon', + 'start_time', + 'end_time' + ]) + example_config_yaml = """ + - data_source: rcmed + dataset_id: 4 + parameter_id: 4 + min_lat: -40 + max_lat: 40 + min_lon: -50 + max_lon: 50 + start_time: YYYY-MM-DDThh:mm:ss + end_time: YYYY-MM-DDThh:mm:ss + + - data_source: rcmed + """ + conf = yaml.load(example_config_yaml) + self.valid_rcmed = conf[0] + self.invalid_rcmed = conf[1] + + def test_valid_rcmed_config(self): + ret = parser._valid_dataset_config_data(self.valid_rcmed) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_invalid_rcmed_config(self, mock_logger): + parser._valid_dataset_config_data(self.invalid_rcmed) + + present_keys = set(self.invalid_rcmed.keys()) + missing_keys = self.required_rcmed_keys - present_keys + missing = sorted(list(missing_keys)) + + error = ( + 'Dataset does not contain required keys. ' + 'The following keys are missing: {}'.format(', '.join(missing)) + ) + mock_logger.error.assert_called_with(error) + + +class TestESGFDatasetConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + self.required_esgf_keys = set([ + 'data_source', + 'dataset_id', + 'variable', + 'esgf_username', + 'esgf_password' + ]) + example_config_yaml = """ + - data_source: esgf + dataset_id: fake dataset id + variable: pr + esgf_username: my esgf username + esgf_password: my esgf password + + - data_source: esgf + """ + conf = yaml.load(example_config_yaml) + self.valid_esgf = conf[0] + self.invalid_esgf = conf[1] + + def test_valid_esgf_conf(self): + ret = parser._valid_dataset_config_data(self.valid_esgf) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_invalid_esgf_conf(self, mock_logger): + parser._valid_dataset_config_data(self.invalid_esgf) + + present_keys = set(self.invalid_esgf.keys()) + missing_keys = self.required_esgf_keys - present_keys + missing = sorted(list(missing_keys)) + + error = ( + 'Dataset does not contain required keys. ' + 'The following keys are missing: {}'.format(', '.join(missing)) + ) + mock_logger.error.assert_called_with(error) + + +class TestDAPDatasetConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + self.required_dap_keys = set(['url', 'variable']) + example_config_yaml = """ + - data_source: dap + url: afakeurl.com + variable: pr + + - data_source: dap + """ + conf = yaml.load(example_config_yaml) + self.valid_dap = conf[0] + self.invalid_dap = conf[1] + + def test_valid_dap_config(self): + ret = parser._valid_dataset_config_data(self.valid_dap) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_invalid_dap_config(self, mock_logger): + parser._valid_dataset_config_data(self.invalid_dap) + + present_keys = set(self.invalid_dap.keys()) + missing_keys = self.required_dap_keys - present_keys + missing = sorted(list(missing_keys)) + + error = ( + 'Dataset does not contain required keys. ' + 'The following keys are missing: {}'.format(', '.join(missing)) + ) + mock_logger.error.assert_called_with(error) + + +class ContourMapConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + valid_contour_config = """ + type: contour + results_indices: + - !!python/tuple [0, 0] + lats: + range_min: -20 + range_max: 20 + range_step: 1 + lons: + range_min: -20 + range_max: 20 + range_step: 1 + output_name: wrf_bias_compared_to_knmi + """ + self.valid_contour = yaml.load(valid_contour_config) + + missing_keys_contour_config = """ + type: contour + """ + self.missing_keys_contour = yaml.load(missing_keys_contour_config) + + self.required_contour_keys = set([ + 'results_indices', + 'lats', + 'lons', + 'output_name' + ]) + + def test_valid_contour(self): + ret = parser._valid_plot_config_data(self.valid_contour) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_missing_keys_contour(self, mock_logger): + ret = parser._valid_plot_config_data(self.missing_keys_contour) + + present_keys = set(self.missing_keys_contour.keys()) + missing_keys = self.required_contour_keys - present_keys + missing = sorted(list(missing_keys)) + + err = ( + 'Plot config does not contain required keys. ' + 'The following keys are missing: {}' + ).format(', '.join(missing)) + mock_logger.error.assert_called_with(err) + + +class TestSubregionPlotConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + valid_subregion_config = """ + type: subregion + lats: + range_min: -20 + range_max: 20 + range_step: 1 + lons: + range_min: -20 + range_max: 20 + range_step: 1 + output_name: fake_plot_name + """ + self.valid_subregion = yaml.load(valid_subregion_config) + + missing_keys_subregion_config = """ + type: subregion + """ + self.missing_keys_subregion = yaml.load(missing_keys_subregion_config) + + self.required_subregion_keys = set([ + 'lats', + 'lons', + 'output_name' + ]) + + def test_valid_subregion(self): + ret = parser._valid_plot_config_data(self.valid_subregion) + self.assertTrue(ret) + + @patch('configuration_parsing.logger') + def test_missing_keys_subregion(self, mock_logger): + ret = parser._valid_plot_config_data(self.missing_keys_subregion) + + present_keys = set(self.missing_keys_subregion.keys()) + missing_keys = self.required_subregion_keys - present_keys + missing = sorted(list(missing_keys)) + + err = ( + 'Plot config does not contain required keys. ' + 'The following keys are missing: {}' + ).format(', '.join(missing)) + mock_logger.error.assert_called_with(err) + + +class TestInvalidPlotConfig(unittest.TestCase): + @classmethod + def setUpClass(self): + bad_plot_type_config = """ + type: NotAPlotType + """ + self.bad_plot_type = yaml.load(bad_plot_type_config) + + missing_plot_type_config = """ + results_indices: + - !!python/tuple [0, 0] + lats: + range_min: -20 + range_max: 20 + range_step: 1 + lons: + range_min: -20 + range_max: 20 + range_step: 1 + output_name: wrf_bias_compared_to_knmi + """ + self.missing_plot_type = yaml.load(missing_plot_type_config) + + missing_subregions_for_plot_type = """ + datasets: + - blah + + metrics: + - blah + + plots: + - type: subregion + results_indices: + - !!python/tuple [0, 0] + lats: + range_min: -20 + range_max: 20 + range_step: 1 + lons: + range_min: -20 + range_max: 20 + range_step: 1 + output_name: wrf_bias_compared_to_knmi + """ + self.missing_subregions = yaml.load(missing_subregions_for_plot_type) + + @patch('configuration_parsing.logger') + def test_invalid_plot_type(self, mock_logger): + ret = parser._valid_plot_config_data(self.bad_plot_type) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Invalid plot type specified.' + ) + + @patch('configuration_parsing.logger') + def test_missing_plot_type(self, mock_logger): + ret = parser._valid_plot_config_data(self.missing_plot_type) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Plot config does not include a type attribute.' + ) + + @patch('configuration_parsing.logger') + def test_missing_subregion(self, mock_logger): + ret = parser._config_is_well_formed(self.missing_subregions) + self.assertFalse(ret) + + mock_logger.error.assert_called_with( + 'Plot config that requires subregion information is present ' + 'in a config file without adequate subregion information ' + 'provided. Please ensure that you have properly supplied 1 or ' + 'more subregion config values.' + ) http://git-wip-us.apache.org/repos/asf/climate/blob/e926a581/ocw_config_runner/tests/test_config_writer.py ---------------------------------------------------------------------- diff --git a/ocw_config_runner/tests/test_config_writer.py b/ocw_config_runner/tests/test_config_writer.py new file mode 100644 index 0000000..f163989 --- /dev/null +++ b/ocw_config_runner/tests/test_config_writer.py @@ -0,0 +1,768 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from mock import patch +import os +import unittest + +from ocw.dataset import Dataset, Bounds +from ocw.evaluation import Evaluation +import ocw.metrics as metrics +import configuration_writer as writer + +import datetime as dt +import numpy as np +import yaml + + +class TestLocalDatasetExportGeneration(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array([10, 12, 14, 16, 18]) + self.lons = np.array([100, 102, 104, 106, 108]) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(300)) + self.values = flat_array.reshape(12, 5, 5) + self.variable = 'var' + self.units = 'units' + self.origin = { + 'source': 'local', + 'path': '/a/fake/path.nc', + 'lat_name': 'a lat name', + 'lon_name': 'a lon name', + 'time_name': 'a time name', + 'elevation_index': 2 + } + self.name = 'name' + + self.dataset = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + origin=self.origin, + name=self.name + ) + + self.exported_info = writer.generate_dataset_config(self.dataset) + + def test_proper_data_source_export(self): + self.assertTrue('data_source' in self.exported_info) + self.assertEqual(self.exported_info['data_source'], + self.origin['source']) + + def test_proper_path_export(self): + self.assertEqual(self.exported_info['path'], self.origin['path']) + + def test_proper_variable_name_export(self): + self.assertEqual(self.exported_info['variable'], self.variable) + + def test_proper_units_name_export(self): + self.assertEqual(self.exported_info['optional_args']['units'], + self.units) + + def test_proper_lats_name_export(self): + self.assertEqual(self.exported_info['optional_args']['lat_name'], + self.origin['lat_name']) + + def test_proper_lons_name_export(self): + self.assertEqual(self.exported_info['optional_args']['lon_name'], + self.origin['lon_name']) + + def test_proper_times_name_export(self): + self.assertEqual(self.exported_info['optional_args']['time_name'], + self.origin['time_name']) + + def test_proper_dataset_name_export(self): + self.assertEqual(self.exported_info['optional_args']['name'], + self.name) + + def test_proper_elevation_index_export(self): + self.assertEqual(self.exported_info['optional_args']['elevation_index'], + self.origin['elevation_index']) + + +class TestRCMEDDatasetExportGeneration(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array([10, 12, 14, 16, 18]) + self.lons = np.array([100, 102, 104, 106, 108]) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(300)) + self.values = flat_array.reshape(12, 5, 5) + self.variable = 'var' + self.units = 'units' + self.origin = { + 'source': 'rcmed', + 'dataset_id': 4, + 'parameter_id': 14 + } + self.name = 'name' + + self.dataset = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + origin=self.origin, + name=self.name + ) + + self.exported_info = writer.generate_dataset_config(self.dataset) + + def test_proper_data_source_export(self): + self.assertTrue('data_source' in self.exported_info) + self.assertEqual(self.exported_info['data_source'], + self.origin['source']) + + def test_proper_dataset_id_export(self): + self.assertEqual(self.exported_info['dataset_id'], + self.origin['dataset_id']) + + def test_proper_parameter_id_export(self): + self.assertEqual(self.exported_info['parameter_id'], + self.origin['parameter_id']) + + def test_proper_min_lat_export(self): + self.assertEqual(self.exported_info['min_lat'], min(self.lats)) + + def test_proper_max_lat_export(self): + self.assertEqual(self.exported_info['max_lat'], max(self.lats)) + + def test_proper_min_lon_export(self): + self.assertEqual(self.exported_info['min_lon'], min(self.lons)) + + def test_proper_max_lon_export(self): + self.assertEqual(self.exported_info['max_lon'], max(self.lons)) + + def test_proper_min_time_export(self): + self.assertEqual(self.exported_info['start_time'], str(min(self.times))) + + def test_proper_max_time_export(self): + self.assertEqual(self.exported_info['end_time'], str(max(self.times))) + + def test_proper_dataset_name_export(self): + self.assertEqual(self.exported_info['optional_args']['name'], + self.name) + + def test_proper_units_name_export(self): + self.assertEqual(self.exported_info['optional_args']['units'], + self.units) + + +class TestESGFDatasetExportGeneration(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array([10, 12, 14, 16, 18]) + self.lons = np.array([100, 102, 104, 106, 108]) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(300)) + self.values = flat_array.reshape(12, 5, 5) + self.variable = 'var' + self.units = 'units' + self.origin = { + 'source': 'esgf', + 'dataset_id': 'esgf dataset id', + 'variable': 'var' + } + self.name = 'name' + + self.dataset = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + origin=self.origin, + name=self.name + ) + + self.exported_info = writer.generate_dataset_config(self.dataset) + + def test_proper_data_source_export(self): + self.assertTrue('data_source' in self.exported_info) + self.assertEqual(self.exported_info['data_source'], + self.origin['source']) + + def test_proper_dataset_id_export(self): + self.assertEqual(self.exported_info['dataset_id'], + self.origin['dataset_id']) + + def test_proper_variable_export(self): + self.assertEqual(self.exported_info['variable'], + self.origin['variable']) + + def test_proper_dummy_username_export(self): + self.assertTrue('esgf_username' in self.exported_info) + + def test_proper_dummy_password_export(self): + self.assertTrue('esgf_password' in self.exported_info) + + def test_proper_dataset_name_export(self): + self.assertEqual(self.exported_info['optional_args']['name'], + self.name) + + def test_proper_units_name_export(self): + self.assertEqual(self.exported_info['optional_args']['units'], + self.units) + + +class TestDAPDatasetExportGeneration(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array([10, 12, 14, 16, 18]) + self.lons = np.array([100, 102, 104, 106, 108]) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(300)) + self.values = flat_array.reshape(12, 5, 5) + self.variable = 'var' + self.units = 'units' + self.origin = { + 'source': 'dap', + 'url': 'a fake url', + } + self.name = 'name' + + self.dataset = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + origin=self.origin, + name=self.name + ) + + self.exported_info = writer.generate_dataset_config(self.dataset) + + def test_proper_data_source_export(self): + self.assertTrue('data_source' in self.exported_info) + self.assertEqual(self.exported_info['data_source'], + self.origin['source']) + + def test_proper_url_export(self): + self.assertEqual(self.exported_info['url'], + self.origin['url']) + + def test_proper_dataset_name_export(self): + self.assertEqual(self.exported_info['optional_args']['name'], + self.name) + + def test_proper_units_name_export(self): + self.assertEqual(self.exported_info['optional_args']['units'], + self.units) + + +class TestDatasetExportFromEvaluation(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array([10, 12, 14, 16, 18]) + self.lons = np.array([100, 102, 104, 106, 108]) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(300)) + self.values = flat_array.reshape(12, 5, 5) + self.variable = 'var' + self.units = 'units' + self.name = 'name' + + self.local_origin = { + 'source': 'local', + 'path': '/a/fake/path.nc', + 'lat_name': 'a lat name', + 'lon_name': 'a lon name', + 'time_name': 'a time name', + 'elevation_index': 2 + } + + self.rcmed_origin = { + 'source': 'rcmed', + 'dataset_id': 4, + 'parameter_id': 14 + } + + self.esgf_origin = { + 'source': 'esgf', + 'dataset_id': 'esgf dataset id', + 'variable': 'var' + } + + self.dap_origin = { + 'source': 'dap', + 'url': 'a fake url', + } + + self.local_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.local_origin + ) + + self.rcmed_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.rcmed_origin + ) + + self.esgf_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.esgf_origin + ) + + self.dap_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.dap_origin + ) + + self.evaluation = Evaluation( + self.local_ds, + [self.rcmed_ds, self.esgf_ds, self.dap_ds], + [] + ) + + def test_contains_only_reference_dataset(self): + new_eval = Evaluation(self.local_ds, [], []) + out = writer.generate_dataset_information(new_eval) + + self.assertTrue('reference' in out) + self.assertTrue('targets' not in out) + + def test_contains_only_target_datasets(self): + new_eval = Evaluation(None, [self.local_ds], []) + out = writer.generate_dataset_information(new_eval) + + self.assertTrue('reference' not in out) + self.assertTrue('targets' in out) + + def test_proper_reference_dataset_export(self): + out = writer.generate_dataset_information(self.evaluation) + + self.assertTrue('reference' in out) + self.assertTrue(out['reference']['data_source'] == 'local') + + def test_proper_target_datasets_export(self): + out = writer.generate_dataset_information(self.evaluation) + + self.assertTrue('targets' in out) + self.assertTrue(type(out['targets']) == type(list())) + self.assertTrue(len(out['targets']) == 3) + + +class TestMetricExportGeneration(unittest.TestCase): + @classmethod + def setUpClass(self): + self.bias = metrics.Bias() + self.tmp_std_dev = metrics.TemporalStdDev() + loaded_metrics = [self.bias, self.tmp_std_dev] + + self.evaluation = Evaluation(None, [], loaded_metrics) + + def test_proper_export_format(self): + out = writer.generate_metric_information(self.evaluation) + + self.assertTrue(type(out) == type(list())) + + for name in out: + self.assertTrue(type(name) == type(str())) + + def test_proper_metric_name_export(self): + out = writer.generate_metric_information(self.evaluation) + + self.assertTrue(self.bias.__class__.__name__ in out) + self.assertTrue(self.tmp_std_dev.__class__.__name__ in out) + + def test_empty_metrics_in_evaluation(self): + new_eval = Evaluation(None, [], []) + out = writer.generate_metric_information(new_eval) + + self.assertTrue(type(out) == type(list())) + self.assertTrue(len(out) == 0) + + +class TestEvaluationSettingsGeneration(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array(range(-10, 10, 1)) + self.lons = np.array(range(-20, 20, 1)) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(9600)) + self.values = flat_array.reshape(12, 20, 40) + + self.dataset = Dataset( + self.lats, + self.lons, + self.times, + self.values, + ) + + self.evaluation = Evaluation(self.dataset, [], []) + + def test_default_data_return(self): + new_eval = Evaluation(None, [], []) + default_output = { + 'temporal_time_delta': 999, + 'spatial_regrid_lats': (-90, 90, 1), + 'spatial_regrid_lons': (-180, 180, 1), + 'subset': [-90, 90, -180, 180, "1500-01-01", "2500-01-01"], + } + + out = writer.generate_evaluation_information(new_eval) + + self.assertEquals(default_output, out) + + def test_handles_only_reference_dataset(self): + new_eval = Evaluation(self.dataset, [], []) + + default_output = { + 'temporal_time_delta': 999, + 'spatial_regrid_lats': (-90, 90, 1), + 'spatial_regrid_lons': (-180, 180, 1), + 'subset': [-90, 90, -180, 180, "1500-01-01", "2500-01-01"], + } + + out = writer.generate_evaluation_information(new_eval) + + self.assertNotEquals(default_output, out) + + def test_handles_only_target_dataset(self): + new_eval = Evaluation(None, [self.dataset], []) + + default_output = { + 'temporal_time_delta': 999, + 'spatial_regrid_lats': (-90, 90, 1), + 'spatial_regrid_lons': (-180, 180, 1), + 'subset': [-90, 90, -180, 180, "1500-01-01", "2500-01-01"], + } + + out = writer.generate_evaluation_information(new_eval) + + self.assertNotEquals(default_output, out) + + def test_daily_temporal_bin(self): + new_times = np.array([dt.datetime(2000, 1, 1, x) for x in range(1, 13)]) + + dataset = Dataset( + self.lats, + self.lons, + new_times, + self.values, + ) + new_eval = Evaluation(dataset, [], []) + + out = writer.generate_evaluation_information(new_eval) + + self.assertEquals(out['temporal_time_delta'], 1) + + def test_monthly_temporal_bin(self): + out = writer.generate_evaluation_information(self.evaluation) + + self.assertEquals(out['temporal_time_delta'], 31) + + def test_yearly_temporal_bin(self): + new_times = np.array([dt.datetime(2000 + x, 1, 1) for x in range(1, 13)]) + + dataset = Dataset( + self.lats, + self.lons, + new_times, + self.values, + ) + new_eval = Evaluation(dataset, [], []) + + out = writer.generate_evaluation_information(new_eval) + + self.assertEquals(out['temporal_time_delta'], 366) + + def test_spatial_regrid_lats(self): + out = writer.generate_evaluation_information(self.evaluation) + + lats = out['spatial_regrid_lats'] + lat_range = np.arange(lats[0], lats[1], lats[2]) + + self.assertTrue(np.array_equal(lat_range, self.lats)) + + def test_spatial_regrid_lons(self): + out = writer.generate_evaluation_information(self.evaluation) + + lons = out['spatial_regrid_lons'] + lat_range = np.arange(lons[0], lons[1], lons[2]) + + self.assertTrue(np.array_equal(lat_range, self.lons)) + + def test_subset_with_single_dataset(self): + out = writer.generate_evaluation_information(self.evaluation) + subset = out['subset'] + + ds_lat_min, ds_lat_max, ds_lon_min, ds_lon_max = self.dataset.spatial_boundaries() + start, end = self.dataset.time_range() + + self.assertEqual(ds_lat_min, subset[0]) + self.assertEqual(ds_lat_max, subset[1]) + self.assertEqual(ds_lon_min, subset[2]) + self.assertEqual(ds_lon_max, subset[3]) + self.assertEquals(str(start), subset[4]) + self.assertEquals(str(end), subset[5]) + + def test_subset_with_multiple_datasets(self): + new_ds = Dataset( + np.arange(0, 20, 1), + self.lons, + self.times, + self.values + ) + new_eval = Evaluation(self.dataset, [new_ds], []) + + out = writer.generate_evaluation_information(new_eval) + subset = out['subset'] + + ds_lat_min, ds_lat_max, ds_lon_min, ds_lon_max = self.dataset.spatial_boundaries() + start, end = self.dataset.time_range() + + self.assertEqual(ds_lat_min, subset[0]) + # Check that we actually used the different max lat value that we + # created by adding 'new_ds'. + self.assertEqual(max(new_ds.lats), subset[1]) + self.assertEqual(ds_lon_min, subset[2]) + self.assertEqual(ds_lon_max, subset[3]) + self.assertEquals(str(start), subset[4]) + self.assertEquals(str(end), subset[5]) + + +class FullExportTest(unittest.TestCase): + @classmethod + def setUpClass(self): + self.lats = np.array([10, 12, 14, 16, 18]) + self.lons = np.array([100, 102, 104, 106, 108]) + self.times = np.array([dt.datetime(2000, x, 1) for x in range(1, 13)]) + flat_array = np.array(range(300)) + self.values = flat_array.reshape(12, 5, 5) + self.variable = 'var' + self.units = 'units' + self.name = 'name' + + self.local_origin = { + 'source': 'local', + 'path': '/a/fake/path.nc', + 'lat_name': 'a lat name', + 'lon_name': 'a lon name', + 'time_name': 'a time name', + 'elevation_index': 2 + } + + self.rcmed_origin = { + 'source': 'rcmed', + 'dataset_id': 4, + 'parameter_id': 14 + } + + self.esgf_origin = { + 'source': 'esgf', + 'dataset_id': 'esgf dataset id', + 'variable': 'var' + } + + self.dap_origin = { + 'source': 'dap', + 'url': 'a fake url', + } + + self.local_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.local_origin + ) + + self.rcmed_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.rcmed_origin + ) + + self.esgf_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.esgf_origin + ) + + self.dap_ds = Dataset( + self.lats, + self.lons, + self.times, + self.values, + variable=self.variable, + units=self.units, + name=self.name, + origin=self.dap_origin + ) + + self.subregions = [ + Bounds(-10, 10, -20, 20), + Bounds(-5, 5, -15, 15) + ] + + self.evaluation = Evaluation( + self.local_ds, + [self.rcmed_ds, self.esgf_ds, self.dap_ds], + [metrics.Bias(), metrics.TemporalStdDev()], + subregions=self.subregions + ) + + @classmethod + def tearDownClass(self): + if os.path.isfile('/tmp/test_config.yaml'): + os.remove('/tmp/test_config.yaml') + + def test_full_export(self): + file_path = '/tmp/test_config.yaml' + writer.export_evaluation_to_config( + self.evaluation, + file_path=file_path + ) + + self.assertTrue(os.path.isfile(file_path)) + + def test_proper_metric_export(self): + file_path = '/tmp/test_config.yaml' + writer.export_evaluation_to_config( + self.evaluation, + file_path=file_path + ) + + data = yaml.load(open(file_path, 'r')) + + self.assertTrue('metrics' in data) + self.assertTrue(type(data['metrics']) == type(list())) + + for metric in self.evaluation.metrics: + self.assertTrue(metric.__class__.__name__ in data['metrics']) + + for metric in self.evaluation.unary_metrics: + self.assertTrue(metric.__class__.__name__ in data['metrics']) + + total_eval_metrics = ( + len(self.evaluation.metrics) + + len(self.evaluation.unary_metrics) + ) + + self.assertTrue(total_eval_metrics, len(data['metrics'])) + + def test_proper_dataset_export(self): + file_path = '/tmp/test_config.yaml' + writer.export_evaluation_to_config( + self.evaluation, + file_path=file_path + ) + + data = yaml.load(open(file_path, 'r')) + + self.assertTrue('datasets' in data) + self.assertTrue('reference' in data['datasets']) + self.assertTrue('targets' in data['datasets']) + + self.assertAlmostEqual( + writer.generate_dataset_information(self.evaluation), + data['datasets'] + ) + + def test_proper_evaluation_setting_export(self): + file_path = '/tmp/test_config.yaml' + writer.export_evaluation_to_config( + self.evaluation, + file_path=file_path + ) + + data = yaml.load(open(file_path, 'r')) + + self.assertTrue('evaluation' in data) + self.assertTrue('temporal_time_delta' in data['evaluation']) + self.assertTrue('spatial_regrid_lats' in data['evaluation']) + self.assertTrue('spatial_regrid_lons' in data['evaluation']) + self.assertTrue('subset' in data['evaluation']) + + self.assertAlmostEqual( + writer.generate_evaluation_information(self.evaluation), + data['evaluation'] + ) + + def test_proper_subregion_export(self): + file_path = '/tmp/test_config.yaml' + writer.export_evaluation_to_config( + self.evaluation, + file_path=file_path + ) + + data = yaml.load(open(file_path, 'r')) + + self.assertTrue('subregions' in data) + + first_bounds = [ + self.subregions[0].lat_min, + self.subregions[0].lat_max, + self.subregions[0].lon_min, + self.subregions[0].lon_max, + ] + second_bounds = [ + self.subregions[1].lat_min, + self.subregions[1].lat_max, + self.subregions[1].lon_min, + self.subregions[1].lon_max, + ] + + self.assertEqual(first_bounds, data['subregions'][0]) + self.assertEqual(second_bounds, data['subregions'][1]) http://git-wip-us.apache.org/repos/asf/climate/blob/e926a581/ocw_config_runner/tests/test_evaluation_creation.py ---------------------------------------------------------------------- diff --git a/ocw_config_runner/tests/test_evaluation_creation.py b/ocw_config_runner/tests/test_evaluation_creation.py new file mode 100644 index 0000000..41f998e --- /dev/null +++ b/ocw_config_runner/tests/test_evaluation_creation.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from mock import patch +import unittest + +import evaluation_creation as eval_create +import ocw.metrics + +import yaml + +class TestMetricLoad(unittest.TestCase): + def test_valid_metric_load(self): + config = yaml.load(""" + metrics: + - Bias + """) + loaded_metrics = [eval_create._load_metric(m)() + for m in config['metrics']] + self.assertTrue(isinstance(loaded_metrics[0], ocw.metrics.Bias)) + + @patch('evaluation_creation.logger') + def test_invalid_metric_load(self, mock_logger): + config = yaml.load(""" + metrics: + - ocw.metrics.Bias + """) + eval_create._load_metric(config['metrics'][0]) + error = ( + 'User-defined metrics outside of the ocw.metrics module ' + 'cannot currently be loaded. If you just wanted a metric ' + 'found in ocw.metrics then do not specify the full ' + 'package and module names. See the documentation for examples.' + ) + mock_logger.error.assert_called_with(error)
