This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch ascending_latitudes in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 3fde4090264e6da81008ef283fe8963547b0dbd4 Author: Eamon Ford <[email protected]> AuthorDate: Tue Aug 11 10:38:08 2020 -0700 SDAP-279: Collection Manager should poll filesystem for file updates (#17) --- .../collection_manager/services/CollectionWatcher.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/collection_manager/collection_manager/services/CollectionWatcher.py b/collection_manager/collection_manager/services/CollectionWatcher.py index 8911806..8f67e16 100644 --- a/collection_manager/collection_manager/services/CollectionWatcher.py +++ b/collection_manager/collection_manager/services/CollectionWatcher.py @@ -1,12 +1,12 @@ import asyncio +import time import logging import os from collections import defaultdict from typing import Dict, Callable, Set, Optional, Awaitable import yaml from watchdog.events import FileSystemEventHandler -from watchdog.observers import Observer -from yaml.scanner import ScannerError +from watchdog.observers.polling import PollingObserver as Observer from collection_manager.entities import Collection from collection_manager.entities.exceptions import RelativePathError, CollectionConfigParsingError, \ @@ -102,9 +102,13 @@ class CollectionWatcher: async def _reload_and_reschedule(self): try: updated_collections = self._get_updated_collections() - for collection in updated_collections: - await self._collection_updated_callback(collection) if len(updated_collections) > 0: + logger.info(f"Scanning files for {len(updated_collections)} collections...") + start = time.perf_counter() + for collection in updated_collections: + await self._collection_updated_callback(collection) + logger.info(f"Finished scanning files in {time.perf_counter() - start} seconds.") + self._unschedule_watches() self._schedule_watches() except CollectionConfigParsingError as e:
