This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch tests
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/tests by this push:
new 92d9aef unit tests
92d9aef is described below
commit 92d9aef5859e6bf1ea47de9d557d489b0626f900
Author: Eamon Ford <[email protected]>
AuthorDate: Fri Jun 12 13:44:15 2020 -0700
unit tests
---
.../collection_manager/entities/Collection.py | 2 +-
.../services/CollectionWatcher.py | 20 +++++++----
.../tests/services/test_CollectionWatcher.py | 39 +++++++++++++++++++---
3 files changed, 49 insertions(+), 12 deletions(-)
diff --git a/collection_manager/collection_manager/entities/Collection.py
b/collection_manager/collection_manager/entities/Collection.py
index d033c69..bea5973 100644
--- a/collection_manager/collection_manager/entities/Collection.py
+++ b/collection_manager/collection_manager/entities/Collection.py
@@ -23,7 +23,7 @@ class Collection(NamedTuple):
variable=properties['variable'],
path=properties['path'],
historical_priority=properties['priority'],
-
forward_processing_priority=properties.get('forward_processing_priority',
+
forward_processing_priority=properties.get('forward-processing-priority',
properties['priority']),
date_to=date_to,
date_from=date_from)
diff --git
a/collection_manager/collection_manager/services/CollectionWatcher.py
b/collection_manager/collection_manager/services/CollectionWatcher.py
index 5261166..b3c4af3 100644
--- a/collection_manager/collection_manager/services/CollectionWatcher.py
+++ b/collection_manager/collection_manager/services/CollectionWatcher.py
@@ -31,16 +31,19 @@ class CollectionWatcher:
self._collections_by_dir: Dict[str, Set[Collection]] = defaultdict(set)
self._observer = Observer()
+ self._granule_watches = set()
+
def start_watching(self):
"""
Start observing filesystem events for added/modified granules or
changes to the Collections configuration file.
When an event occurs, call the appropriate callback that was passed in
during instantiation.
:return: None
"""
-
self._observer.schedule(_CollectionEventHandler(file_path=self._collections_path,
callback=self._refresh),
- os.path.dirname(self._collections_path))
+ self._observer.schedule(
+ _CollectionEventHandler(file_path=self._collections_path,
callback=self._reload_and_reschedule),
+ os.path.dirname(self._collections_path))
self._observer.start()
- self._refresh()
+ self._reload_and_reschedule()
def collections(self) -> Set[Collection]:
"""
@@ -85,21 +88,26 @@ class CollectionWatcher:
self._load_collections()
return self.collections() - old_collections
- def _refresh(self):
+ def _reload_and_reschedule(self):
try:
for collection in self._get_updated_collections():
self._collection_updated_callback(collection)
- self._observer.unschedule_all()
+ self._unschedule_watches()
self._schedule_watches()
except YamlParsingError as e:
logger.error(e)
+ def _unschedule_watches(self):
+ for watch in self._granule_watches:
+ self._observer.unschedule(watch)
+ self._granule_watches.clear()
+
def _schedule_watches(self):
for directory, collections in self._collections_by_dir.items():
granule_event_handler =
_GranuleEventHandler(self._granule_updated_callback, collections)
# Note: the Watchdog library does not schedule a new watch
# if one is already scheduled for the same directory
- self._observer.schedule(granule_event_handler, directory)
+
self._granule_watches.add(self._observer.schedule(granule_event_handler,
directory))
class _CollectionEventHandler(FileSystemEventHandler):
diff --git a/collection_manager/tests/services/test_CollectionWatcher.py
b/collection_manager/tests/services/test_CollectionWatcher.py
index cb9d38a..a1d8022 100644
--- a/collection_manager/tests/services/test_CollectionWatcher.py
+++ b/collection_manager/tests/services/test_CollectionWatcher.py
@@ -1,8 +1,9 @@
import os
+import tempfile
import unittest
from datetime import datetime
from unittest.mock import Mock
-
+import time
from collection_manager.entities import Collection
from collection_manager.entities.exceptions import YamlParsingError,
CollectionConfigFileNotFoundError
from collection_manager.services import CollectionWatcher
@@ -23,7 +24,7 @@ class TestCollectionWatcher(unittest.TestCase):
}
}
flattened_collections = collection_watcher.collections()
- self.assertEquals(len(flattened_collections), 4)
+ self.assertEqual(len(flattened_collections), 4)
def test_load_collections_loads_all_collections(self):
collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
@@ -72,6 +73,34 @@ class TestCollectionWatcher(unittest.TestCase):
self.assertEquals(len(updated_collections), 1)
- def test_schedule_watches(self):
- collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
- collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+ def test_collection_callback_is_called(self):
+ with tempfile.NamedTemporaryFile("w+b", buffering=0) as
collections_config:
+ granule_dir = tempfile.TemporaryDirectory()
+ collections_str = f"""
+collections:
+- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
+ path: {granule_dir.name}
+ variable: lwe_thickness
+ priority: 1
+ forward-processing-priority: 5
+ """
+ collections_config.write(collections_str.encode("utf-8"))
+
+ collection_callback = Mock()
+ collection_watcher = CollectionWatcher(collections_config.name,
collection_callback, Mock())
+ collection_watcher.start_watching()
+
+ collections_str = f"""
+- id: TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_LAND
+ path: {granule_dir.name}
+ variable: lwe_thickness
+ priority: 10
+ forward-processing-priority: 5
+ """
+ collections_config.write(collections_str.encode("utf-8"))
+ time.sleep(1)
+
+ self.assertEqual(2, collection_callback.call_count)
+
+ def test_granule_callback_is_called(self):
+ ...
\ No newline at end of file
