This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new 1e492cf SDAP-279: Collection Manager should poll filesystem for file
updates (#17)
1e492cf is described below
commit 1e492cf050209fb3e724ca9d92a2cb5fc6d4910a
Author: Eamon Ford <[email protected]>
AuthorDate: Tue Aug 11 10:38:08 2020 -0700
SDAP-279: Collection Manager should poll filesystem for file updates (#17)
---
.../collection_manager/services/CollectionWatcher.py | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git
a/collection_manager/collection_manager/services/CollectionWatcher.py
b/collection_manager/collection_manager/services/CollectionWatcher.py
index 8911806..8f67e16 100644
--- a/collection_manager/collection_manager/services/CollectionWatcher.py
+++ b/collection_manager/collection_manager/services/CollectionWatcher.py
@@ -1,12 +1,12 @@
import asyncio
+import time
import logging
import os
from collections import defaultdict
from typing import Dict, Callable, Set, Optional, Awaitable
import yaml
from watchdog.events import FileSystemEventHandler
-from watchdog.observers import Observer
-from yaml.scanner import ScannerError
+from watchdog.observers.polling import PollingObserver as Observer
from collection_manager.entities import Collection
from collection_manager.entities.exceptions import RelativePathError,
CollectionConfigParsingError, \
@@ -102,9 +102,13 @@ class CollectionWatcher:
async def _reload_and_reschedule(self):
try:
updated_collections = self._get_updated_collections()
- for collection in updated_collections:
- await self._collection_updated_callback(collection)
if len(updated_collections) > 0:
+ logger.info(f"Scanning files for {len(updated_collections)}
collections...")
+ start = time.perf_counter()
+ for collection in updated_collections:
+ await self._collection_updated_callback(collection)
+ logger.info(f"Finished scanning files in {time.perf_counter()
- start} seconds.")
+
self._unschedule_watches()
self._schedule_watches()
except CollectionConfigParsingError as e:
