This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch configure-processors
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/configure-processors by this
push:
new 3d01709 fix unit tests
3d01709 is described below
commit 3d01709ba008affeb7007edd61c8655e059cda4a
Author: Eamon Ford <[email protected]>
AuthorDate: Wed Aug 19 12:26:13 2020 -0700
fix unit tests
---
.../collection_manager/entities/Collection.py | 6 +-
.../services/CollectionProcessor.py | 4 +-
.../tests/entities/test_Collection.py | 40 +++++++++--
collection_manager/tests/resources/collections.yml | 36 +++++++++-
.../tests/resources/collections_alternate.yml | 34 +++++++++-
.../tests/services/test_CollectionProcessor.py | 79 ++++++++++++++--------
.../tests/services/test_CollectionWatcher.py | 59 +++++++++++++---
7 files changed, 206 insertions(+), 52 deletions(-)
diff --git a/collection_manager/collection_manager/entities/Collection.py
b/collection_manager/collection_manager/entities/Collection.py
index 0ca12d9..ca5bbc1 100644
--- a/collection_manager/collection_manager/entities/Collection.py
+++ b/collection_manager/collection_manager/entities/Collection.py
@@ -3,7 +3,7 @@ from dataclasses import dataclass
from datetime import datetime
from fnmatch import fnmatch
from glob import glob
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, FrozenSet
from collection_manager.entities.exceptions import MissingValueCollectionError
@@ -12,8 +12,8 @@ from collection_manager.entities.exceptions import
MissingValueCollectionError
class Collection:
dataset_id: str
projection: str
- dimension_names: Dict[str, str]
- slices: Dict[str, int]
+ dimension_names: frozenset
+ slices: frozenset
path: str
historical_priority: int
forward_processing_priority: Optional[int] = None
diff --git
a/collection_manager/collection_manager/services/CollectionProcessor.py
b/collection_manager/collection_manager/services/CollectionProcessor.py
index 88f512b..ce0aea4 100644
--- a/collection_manager/collection_manager/services/CollectionProcessor.py
+++ b/collection_manager/collection_manager/services/CollectionProcessor.py
@@ -62,7 +62,7 @@ class CollectionProcessor:
f"collection '{collection.dataset_id}'. Skipping.")
return
- dataset_config = self._fill_template(granule, collection,
config_template=self._config_template)
+ dataset_config = self._fill_template(granule, collection)
await self._publisher.publish_message(body=dataset_config,
priority=use_priority)
await history_manager.push(granule)
@@ -77,7 +77,7 @@ class CollectionProcessor:
return self._history_manager_cache[dataset_id]
@staticmethod
- def _fill_template(granule_path: str, collection: Collection,
config_template: str) -> str:
+ def _fill_template(granule_path: str, collection: Collection) -> str:
config_dict = {
'granule': {
'resource': granule_path
diff --git a/collection_manager/tests/entities/test_Collection.py
b/collection_manager/tests/entities/test_Collection.py
index 2b83fcb..7e56c9d 100644
--- a/collection_manager/tests/entities/test_Collection.py
+++ b/collection_manager/tests/entities/test_Collection.py
@@ -79,7 +79,9 @@ class TestCollection(unittest.TestCase):
pattern = os.path.join(directory, "test_*.nc")
collection = Collection(dataset_id="test_dataset",
path=pattern,
- variable="test_variable",
+ projection="Grid",
+ slices={},
+ dimension_names={},
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -92,7 +94,9 @@ class TestCollection(unittest.TestCase):
pattern = os.path.join(directory, "test_*.nc")
collection = Collection(dataset_id="test_dataset",
path=pattern,
- variable="test_variable",
+ projection="Grid",
+ slices={},
+ dimension_names={},
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -103,8 +107,14 @@ class TestCollection(unittest.TestCase):
def test_from_dict(self):
collection_dict = {
'id': 'test_id',
- 'variable': 'test_var',
'path': '/some/path',
+ 'projection': 'Grid',
+ 'dimensionNames': {
+ 'latitude': 'lat',
+ 'longitude': 'lon',
+ 'variable': 'test_var'
+ },
+ 'slices': {'lat': 30, 'lon': 30, 'time': 1},
'priority': 1,
'forward-processing-priority': 2,
'from': '2020-01-01T00:00:00+00:00',
@@ -112,7 +122,13 @@ class TestCollection(unittest.TestCase):
}
expected_collection = Collection(dataset_id='test_id',
- variable='test_var',
+ projection="Grid",
+ slices=frozenset([('lat', 30),
('lon', 30), ('time', 1)]),
+ dimension_names=frozenset([
+ ('latitude', 'lat'),
+ ('longitude', 'lon'),
+ ('variable', 'test_var')
+ ]),
path='/some/path',
historical_priority=1,
forward_processing_priority=2,
@@ -124,13 +140,25 @@ class TestCollection(unittest.TestCase):
def test_from_dict_missing_optional_values(self):
collection_dict = {
'id': 'test_id',
- 'variable': 'test_var',
+ 'projection': 'Grid',
+ 'dimensionNames': {
+ 'latitude': 'lat',
+ 'longitude': 'lon',
+ 'variable': 'test_var'
+ },
+ 'slices': {'lat': 30, 'lon': 30, 'time': 1},
'path': '/some/path',
'priority': 3
}
expected_collection = Collection(dataset_id='test_id',
- variable='test_var',
+ projection="Grid",
+ slices=frozenset([('lat', 30),
('lon', 30), ('time', 1)]),
+ dimension_names=frozenset([
+ ('latitude', 'lat'),
+ ('longitude', 'lon'),
+ ('variable', 'test_var')
+ ]),
path='/some/path',
historical_priority=3,
forward_processing_priority=None,
diff --git a/collection_manager/tests/resources/collections.yml
b/collection_manager/tests/resources/collections.yml
index 89524ec..44f795b 100644
--- a/collection_manager/tests/resources/collections.yml
+++ b/collection_manager/tests/resources/collections.yml
@@ -1,17 +1,47 @@
collections:
- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
path: /opt/data/grace/*land*.nc
- variable: lwe_thickness
priority: 1
forward-processing-priority: 5
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
+
- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_OCEAN
path: /opt/data/grace/*ocean*.nc
- variable: lwe_thickness
priority: 2
forward-processing-priority: 6
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
+
- id: AVHRR_OI-NCEI-L4-GLOB-v2.0
path: /opt/data/avhrr/*.nc
- variable: analysed_sst
priority: 1
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: analysed_sst
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
+
diff --git a/collection_manager/tests/resources/collections_alternate.yml
b/collection_manager/tests/resources/collections_alternate.yml
index 3d7da95..f9dabda 100644
--- a/collection_manager/tests/resources/collections_alternate.yml
+++ b/collection_manager/tests/resources/collections_alternate.yml
@@ -1,17 +1,45 @@
collections:
- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
path: /opt/data/grace/*land*.nc
- variable: lwe_thickness
priority: 1
forward-processing-priority: 5
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
- id: ID_CHANGED
path: /opt/data/grace/*ocean*.nc
- variable: lwe_thickness
priority: 2
forward-processing-priority: 6
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
- id: AVHRR_OI-NCEI-L4-GLOB-v2.0
path: /opt/data/avhrr/*.nc
- variable: analysed_sst
priority: 1
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: analysed_sst
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
+
diff --git a/collection_manager/tests/services/test_CollectionProcessor.py
b/collection_manager/tests/services/test_CollectionProcessor.py
index a7059d6..96e63ac 100644
--- a/collection_manager/tests/services/test_CollectionProcessor.py
+++ b/collection_manager/tests/services/test_CollectionProcessor.py
@@ -1,4 +1,5 @@
import tempfile
+import yaml
import unittest
from unittest import mock
@@ -35,34 +36,48 @@ class TestCollectionProcessor(unittest.TestCase):
self.assertIsNot(collection_processor._get_history_manager('bar'),
history_manager)
def test_fill_template(self):
- template = """
- granule:
- resource: {{granule}}
- processors:
- - name: GridReadingProcessor
- variable_to_read: {{variable}}
- - name: tileSummary
- dataset_name: {{dataset_id}}
- """
-
- expected = """
- granule:
- resource: /granules/test_granule.nc
- processors:
- - name: GridReadingProcessor
- variable_to_read: test_variable
- - name: tileSummary
- dataset_name: test_dataset
- """
+ expected = {
+ 'granule': {
+ 'resource': '/granules/test_granule.nc'
+ },
+ 'processors': [
+ {
+ 'latitude': 'lat',
+ 'longitude': 'lon',
+ 'name': 'Grid',
+ 'variable': 'test_var'
+ },
+ {'name': 'emptyTileFilter'},
+ {'name': 'kelvinToCelsius'},
+ {'dataset_name': 'test_dataset', 'name': 'tileSummary'},
+ {'name': 'generateTileId'}
+ ],
+ 'slicer': {
+ 'dimension_step_sizes': {
+ 'lat': 30,
+ 'lon': 30,
+ 'time': 1
+ },
+ 'name': 'sliceFileByStepSize'
+ }
+ }
collection = Collection(dataset_id="test_dataset",
path="/granules/test*.nc",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset([('lat', 30), ('lon', 30),
('time', 1)]),
+ dimension_names=frozenset([
+ ('latitude', 'lat'),
+ ('longitude', 'lon'),
+ ('variable', 'test_var')
+ ]),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
date_to=None)
- filled =
CollectionProcessor._fill_template("/granules/test_granule.nc", collection,
template)
- self.assertEqual(filled, expected)
+ filled =
CollectionProcessor._fill_template("/granules/test_granule.nc", collection)
+ generated_yaml = yaml.load(filled, Loader=yaml.FullLoader)
+
+ self.assertEqual(expected, generated_yaml)
@async_test
@mock.patch('collection_manager.services.history_manager.FileIngestionHistory',
new_callable=AsyncMock)
@@ -75,7 +90,9 @@ class TestCollectionProcessor(unittest.TestCase):
collection_processor = CollectionProcessor(mock_publisher,
mock_history_builder)
collection = Collection(dataset_id="test_dataset",
path="test_path",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -100,7 +117,9 @@ class TestCollectionProcessor(unittest.TestCase):
collection_processor = CollectionProcessor(mock_publisher,
mock_history_builder)
collection = Collection(dataset_id="test_dataset",
path="test_path",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -123,7 +142,9 @@ class TestCollectionProcessor(unittest.TestCase):
collection_processor = CollectionProcessor(mock_publisher,
mock_history_builder)
collection = Collection(dataset_id="test_dataset",
path="test_path",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
date_from=None,
date_to=None)
@@ -144,7 +165,9 @@ class TestCollectionProcessor(unittest.TestCase):
collection_processor = CollectionProcessor(mock_publisher,
mock_history_builder)
collection = Collection(dataset_id="test_dataset",
path="test_path",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -165,7 +188,9 @@ class TestCollectionProcessor(unittest.TestCase):
collection_processor = CollectionProcessor(mock_publisher,
mock_history_builder)
collection = Collection(dataset_id="test_dataset",
path="test_path",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
diff --git a/collection_manager/tests/services/test_CollectionWatcher.py
b/collection_manager/tests/services/test_CollectionWatcher.py
index 0dc924b..e6bf15f 100644
--- a/collection_manager/tests/services/test_CollectionWatcher.py
+++ b/collection_manager/tests/services/test_CollectionWatcher.py
@@ -87,7 +87,9 @@ class TestCollectionWatcher(unittest.TestCase):
collection = Collection(dataset_id="test_dataset",
path="/absolute/path",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -100,7 +102,9 @@ class TestCollectionWatcher(unittest.TestCase):
collection = Collection(dataset_id="test_dataset",
path="relative/path",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -113,7 +117,9 @@ class TestCollectionWatcher(unittest.TestCase):
collection = Collection(dataset_id="test_dataset",
path="/resources/*.nc",
- variable="test_variable",
+ projection="Grid",
+ slices=frozenset(),
+ dimension_names=frozenset(),
historical_priority=1,
forward_processing_priority=2,
date_from=None,
@@ -127,9 +133,19 @@ class TestCollectionWatcher(unittest.TestCase):
collections_str = f"""collections:
- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
path: {granule_dir.name}
- variable: lwe_thickness
priority: 1
- forward-processing-priority: 5"""
+ forward-processing-priority: 5
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
+ """
collections_config.write(collections_str.encode("utf-8"))
collection_callback = AsyncMock()
@@ -143,9 +159,18 @@ class TestCollectionWatcher(unittest.TestCase):
collections_str = f"""
- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
path: {granule_dir.name}
- variable: lwe_thickness
priority: 10
forward-processing-priority: 5
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
"""
collections_config.write(collections_str.encode("utf-8"))
@@ -163,9 +188,18 @@ class TestCollectionWatcher(unittest.TestCase):
collections:
- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
path: {granule_dir.name}
- variable: lwe_thickness
priority: 1
forward-processing-priority: 5
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
"""
collections_config.write(collections_str.encode("utf-8"))
@@ -187,9 +221,18 @@ collections:
collections:
- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
path: {granule_dir.name}
- variable: lwe_thickness
priority: 1
forward-processing-priority: 5
+ projection: Grid
+ dimensionNames:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable: lwe_thickness
+ slices:
+ time: 1
+ lat: 30
+ lon: 30
"""
collections_config.write(collections_str.encode("utf-8"))
new_granule = open(os.path.join(granule_dir.name, 'test.nc'), "w+")
